v0.16 migration and other fixes
This commit is contained in:
Maurus Decimus
166
resources/scripts/imap-log-sanitizer.py
Normal file
166
resources/scripts/imap-log-sanitizer.py
Normal file
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
IMAP Log sanitizer - Extracts and groups IMAP transactions from log files
|
||||
"""
|
||||
|
||||
# SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
|
||||
import re
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
import argparse
|
||||
|
||||
def unescape_imap_content(content):
|
||||
|
||||
if content.startswith('"') and content.endswith('"'):
|
||||
content = content[1:-1]
|
||||
|
||||
replacements = {
|
||||
'\\r\\n': '\r\n',
|
||||
'\\n': '\n',
|
||||
'\\r': '\r',
|
||||
'\\t': '\t',
|
||||
'\\"': '"',
|
||||
'\\\\': '\\'
|
||||
}
|
||||
|
||||
for escaped, unescaped in replacements.items():
|
||||
content = content.replace(escaped, unescaped)
|
||||
|
||||
return content
|
||||
|
||||
def parse_imap_log_line(line):
|
||||
|
||||
pattern = r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)\s+TRACE\s+Raw IMAP\s+(input received|output sent)\s+.*?remoteIp\s*=\s*([^,]+),\s*remotePort\s*=\s*(\d+).*?contents\s*=\s*(.+)$'
|
||||
|
||||
match = re.search(pattern, line)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
timestamp, direction, remote_ip, remote_port, contents = match.groups()
|
||||
|
||||
return {
|
||||
'timestamp': timestamp,
|
||||
'direction': direction,
|
||||
'remote_ip': remote_ip.strip(),
|
||||
'remote_port': int(remote_port),
|
||||
'contents': unescape_imap_content(contents.strip()),
|
||||
'raw_line': line.strip()
|
||||
}
|
||||
|
||||
def group_by_connection(log_entries):
|
||||
|
||||
connections = defaultdict(list)
|
||||
|
||||
for entry in log_entries:
|
||||
if entry:
|
||||
key = f"{entry['remote_ip']}:{entry['remote_port']}"
|
||||
connections[key].append(entry)
|
||||
|
||||
for key in connections:
|
||||
connections[key].sort(key=lambda x: x['timestamp'])
|
||||
|
||||
return dict(connections)
|
||||
|
||||
def format_imap_transaction(entries):
|
||||
|
||||
transaction = []
|
||||
|
||||
for entry in entries:
|
||||
direction_symbol = "C: " if "input received" in entry['direction'] else "S: "
|
||||
timestamp = entry['timestamp']
|
||||
content = entry['contents']
|
||||
|
||||
if content.endswith('\\r\\n') or content.endswith('\r\n'):
|
||||
content = content.rstrip('\\r\\n\r\n')
|
||||
|
||||
transaction.append(f"[{timestamp}] {direction_symbol}{content}")
|
||||
|
||||
return transaction
|
||||
|
||||
def write_output_file(connections, output_file):
|
||||
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
f.write("IMAP Transaction Log Analysis\n")
|
||||
f.write("=" * 50 + "\n\n")
|
||||
|
||||
for connection_key, entries in connections.items():
|
||||
f.write(f"Connection: {connection_key}\n")
|
||||
f.write("-" * 30 + "\n")
|
||||
f.write(f"Total messages: {len(entries)}\n")
|
||||
f.write(f"Duration: {entries[0]['timestamp']} to {entries[-1]['timestamp']}\n\n")
|
||||
|
||||
transaction = format_imap_transaction(entries)
|
||||
for line in transaction:
|
||||
f.write(line + "\n")
|
||||
|
||||
f.write("\n" + "=" * 50 + "\n\n")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Parse IMAP log files and group transactions by connection')
|
||||
parser.add_argument('input_file', help='Input log file path')
|
||||
parser.add_argument('-o', '--output', default='imap_transactions.txt',
|
||||
help='Output file path (default: imap_transactions.txt)')
|
||||
parser.add_argument('-j', '--json', action='store_true',
|
||||
help='Also output raw data as JSON')
|
||||
parser.add_argument('-v', '--verbose', action='store_true',
|
||||
help='Enable verbose output')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
print(f"Reading log file: {args.input_file}")
|
||||
|
||||
log_entries = []
|
||||
imap_line_count = 0
|
||||
|
||||
try:
|
||||
with open(args.input_file, 'r', encoding='utf-8') as f:
|
||||
for line_num, line in enumerate(f, 1):
|
||||
if 'Raw IMAP' in line:
|
||||
imap_line_count += 1
|
||||
parsed_entry = parse_imap_log_line(line)
|
||||
if parsed_entry:
|
||||
log_entries.append(parsed_entry)
|
||||
elif args.verbose:
|
||||
print(f"Warning: Could not parse line {line_num}: {line.strip()}")
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File '{args.input_file}' not found")
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"Error reading file: {e}")
|
||||
return 1
|
||||
|
||||
if args.verbose:
|
||||
print(f"Found {imap_line_count} Raw IMAP lines")
|
||||
print(f"Successfully parsed {len(log_entries)} entries")
|
||||
|
||||
connections = group_by_connection(log_entries)
|
||||
|
||||
if args.verbose:
|
||||
print(f"Found {len(connections)} unique connections:")
|
||||
for conn_key, entries in connections.items():
|
||||
print(f" {conn_key}: {len(entries)} messages")
|
||||
|
||||
try:
|
||||
write_output_file(connections, args.output)
|
||||
print(f"IMAP transactions written to: {args.output}")
|
||||
|
||||
if args.json:
|
||||
json_file = args.output.rsplit('.', 1)[0] + '.json'
|
||||
with open(json_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(connections, f, indent=2, ensure_ascii=False)
|
||||
print(f"Raw data written to: {json_file}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error writing output: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
1563
resources/scripts/migrate_v016.py
Normal file
1563
resources/scripts/migrate_v016.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,10 @@ This script removes SEL code from the Stalwart codebase by:
|
||||
Usage: python ossify.py <stalwart_repository>/crates
|
||||
"""
|
||||
|
||||
# SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
@@ -16,171 +20,142 @@ import argparse
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
|
||||
def find_first_comment_block(content: str) -> Optional[str]:
|
||||
"""
|
||||
Find the first comment block in a Rust file.
|
||||
Returns the comment content or None if no comment block is found.
|
||||
"""
|
||||
# Remove leading whitespace and find the first comment
|
||||
|
||||
lines = content.strip().split('\n')
|
||||
|
||||
|
||||
if not lines:
|
||||
return None
|
||||
|
||||
|
||||
first_line = lines[0].strip()
|
||||
|
||||
# Check for block comment starting with /*
|
||||
|
||||
if first_line.startswith('/*'):
|
||||
comment_lines = []
|
||||
in_comment = True
|
||||
|
||||
|
||||
for line in lines:
|
||||
if in_comment:
|
||||
comment_lines.append(line)
|
||||
if '*/' in line:
|
||||
break
|
||||
|
||||
|
||||
return '\n'.join(comment_lines)
|
||||
|
||||
# Check for line comments starting with //
|
||||
|
||||
elif first_line.startswith('//'):
|
||||
comment_lines = []
|
||||
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped.startswith('//'):
|
||||
comment_lines.append(line)
|
||||
elif stripped == '':
|
||||
comment_lines.append(line) # Keep empty lines within comment block
|
||||
comment_lines.append(line)
|
||||
else:
|
||||
break # Stop at first non-comment, non-empty line
|
||||
|
||||
break
|
||||
|
||||
return '\n'.join(comment_lines)
|
||||
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def should_remove_file(file_path: str) -> bool:
|
||||
"""
|
||||
Check if a .rs file should be completely removed based on its first comment.
|
||||
Returns True if the file contains "SPDX-License-Identifier: LicenseRef-SEL" in the first comment.
|
||||
"""
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
|
||||
first_comment = find_first_comment_block(content)
|
||||
if first_comment and 'SPDX-License-Identifier: LicenseRef-SEL' in first_comment:
|
||||
return True
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading file {file_path}: {e}")
|
||||
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def remove_proprietary_snippets(content: str) -> Tuple[str, int]:
|
||||
"""
|
||||
Remove proprietary snippets from file content.
|
||||
Returns tuple of (modified_content, number_of_snippets_removed)
|
||||
"""
|
||||
|
||||
snippets_removed = 0
|
||||
|
||||
# Pattern to match SPDX snippets that contain LicenseRef-SEL
|
||||
# We look for SPDX-SnippetBegin, then check if the snippet contains LicenseRef-SEL,
|
||||
# and if so, remove everything until SPDX-SnippetEnd
|
||||
|
||||
|
||||
lines = content.split('\n')
|
||||
result_lines = []
|
||||
i = 0
|
||||
|
||||
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
|
||||
# Check if this line starts a snippet
|
||||
|
||||
if '// SPDX-SnippetBegin' in line:
|
||||
# Look ahead to see if this snippet contains LicenseRef-SEL
|
||||
|
||||
snippet_start = i
|
||||
snippet_lines = []
|
||||
j = i
|
||||
|
||||
# Collect the snippet lines until we find SnippetEnd or reach end of file
|
||||
|
||||
while j < len(lines):
|
||||
snippet_lines.append(lines[j])
|
||||
if '// SPDX-SnippetEnd' in lines[j]:
|
||||
break
|
||||
j += 1
|
||||
|
||||
# Check if this snippet contains LicenseRef-SEL
|
||||
|
||||
snippet_content = '\n'.join(snippet_lines)
|
||||
if 'SPDX-License-Identifier: LicenseRef-SEL' in snippet_content:
|
||||
# Remove this snippet
|
||||
|
||||
snippets_removed += 1
|
||||
i = j + 1 # Skip past the SnippetEnd line
|
||||
i = j + 1
|
||||
continue
|
||||
else:
|
||||
# Keep this snippet as it's not proprietary
|
||||
|
||||
result_lines.append(line)
|
||||
i += 1
|
||||
else:
|
||||
result_lines.append(line)
|
||||
i += 1
|
||||
|
||||
|
||||
return '\n'.join(result_lines), snippets_removed
|
||||
|
||||
|
||||
def process_rust_file(file_path: str, dry_run: bool = False) -> dict:
|
||||
"""
|
||||
Process a single Rust file, removing proprietary content.
|
||||
Returns a dictionary with processing results.
|
||||
"""
|
||||
|
||||
result = {
|
||||
'file': file_path,
|
||||
'action': 'none',
|
||||
'snippets_removed': 0,
|
||||
'error': None
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
# Check if the entire file should be removed
|
||||
|
||||
if should_remove_file(file_path):
|
||||
result['action'] = 'file_removed'
|
||||
if not dry_run:
|
||||
os.remove(file_path)
|
||||
return result
|
||||
|
||||
# Process snippets in the file
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
original_content = f.read()
|
||||
|
||||
|
||||
modified_content, snippets_removed = remove_proprietary_snippets(original_content)
|
||||
|
||||
|
||||
if snippets_removed > 0:
|
||||
result['action'] = 'snippets_removed'
|
||||
result['snippets_removed'] = snippets_removed
|
||||
|
||||
|
||||
if not dry_run:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.write(modified_content)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
result['error'] = str(e)
|
||||
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def find_rust_files(directory: str) -> List[str]:
|
||||
"""Find all .rs files in the given directory recursively."""
|
||||
|
||||
rust_files = []
|
||||
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
for file in files:
|
||||
if file.endswith('.rs'):
|
||||
rust_files.append(os.path.join(root, file))
|
||||
|
||||
return rust_files
|
||||
|
||||
return rust_files
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -200,66 +175,64 @@ def main():
|
||||
action='store_true',
|
||||
help='Show detailed output for each file'
|
||||
)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
if not os.path.isdir(args.directory):
|
||||
print(f"Error: {args.directory} is not a valid directory")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
print(f"Processing Rust files in: {args.directory}")
|
||||
if args.dry_run:
|
||||
print("DRY RUN MODE - No changes will be made")
|
||||
print()
|
||||
|
||||
|
||||
rust_files = find_rust_files(args.directory)
|
||||
|
||||
|
||||
if not rust_files:
|
||||
print("No .rs files found in the specified directory")
|
||||
return
|
||||
|
||||
|
||||
print(f"Found {len(rust_files)} Rust files")
|
||||
print()
|
||||
|
||||
|
||||
files_removed = 0
|
||||
files_with_snippets_removed = 0
|
||||
total_snippets_removed = 0
|
||||
errors = []
|
||||
|
||||
|
||||
for file_path in rust_files:
|
||||
result = process_rust_file(file_path, args.dry_run)
|
||||
|
||||
|
||||
if result['error']:
|
||||
errors.append(f"{file_path}: {result['error']}")
|
||||
continue
|
||||
|
||||
|
||||
if result['action'] == 'file_removed':
|
||||
files_removed += 1
|
||||
if args.verbose or args.dry_run:
|
||||
action_text = "Would remove" if args.dry_run else "Removed"
|
||||
print(f"{action_text} file: {file_path}")
|
||||
|
||||
|
||||
elif result['action'] == 'snippets_removed':
|
||||
files_with_snippets_removed += 1
|
||||
total_snippets_removed += result['snippets_removed']
|
||||
if args.verbose or args.dry_run:
|
||||
action_text = "Would remove" if args.dry_run else "Removed"
|
||||
print(f"{action_text} {result['snippets_removed']} snippet(s) from: {file_path}")
|
||||
|
||||
# Summary
|
||||
|
||||
print("\nSummary:")
|
||||
action_text = "Would be" if args.dry_run else "Were"
|
||||
print(f"- {files_removed} files {action_text.lower()} completely removed")
|
||||
print(f"- {total_snippets_removed} proprietary snippets {action_text.lower()} removed from {files_with_snippets_removed} files")
|
||||
|
||||
|
||||
if errors:
|
||||
print(f"- {len(errors)} errors occurred:")
|
||||
for error in errors:
|
||||
print(f" {error}")
|
||||
|
||||
|
||||
if args.dry_run:
|
||||
print("\nRun without --dry-run to apply changes")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user