commit 748d2b26a42c976e0137045a2f19bdfcc7b474e8 Author: luorijun Date: Sat Dec 13 13:18:58 2025 +0800 init commit diff --git a/.ansible/.lock b/.ansible/.lock new file mode 100644 index 0000000..e69de29 diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..251d4c0 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "ansible.python.interpreterPath": "/usr/bin/python" +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a4be565 --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# Ansible 日志管理自动化 + +## 目录结构 + +``` +ansible/ +├── inventory/ +│ └── hosts.yaml # 服务器清单 +├── playbooks/ +│ └── compress_logs.yaml # 日志压缩 playbook +├── scripts/ +│ └── compress_logs.sh # 快捷执行脚本 +└── README.md +``` + +## 快速开始 + +### 基本用法 + +```bash +# 必须指定年月日参数 +ansible-playbook -i inventory/hosts.yaml playbooks/compress_logs.yaml \ + -e "year=2025 month=11 start_day=21 end_day=30" + +# 使用便捷脚本 +./scripts/compress_logs.sh -y 2025 -m 11 -s 21 -e 30 + +# 指定单个服务器 +ansible-playbook -i inventory/hosts.yaml playbooks/compress_logs.yaml \ + --limit jhlog -e "year=2025 month=11 start_day=21 end_day=30" +``` + +**默认行为**: +- 压缩日志到 tar.gz +- 下载到 `/mnt/d/logs////` +- 删除原始文件夹和远程压缩包 +- 支持断点续传 + +## 可配置参数 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `year` | **必填** | 年份 | +| `month` | **必填** | 月份 | +| `start_day` | **必填** | 起始日期 | +| `end_day` | **必填** | 结束日期 | +| `log_base_path` | `/data/roslog/roslog` | 日志路径 | +| `local_download_path` | `/mnt/d/logs` | 本地下载路径 | +| `remove_after_compress` | `true` | 删除原文件夹 | +| `download_to_local` | `true` | 下载到本地 | +| `remove_local_archives` | `true` | 删除远程压缩包 | + +## 常用场景 + +### 只压缩不删除 + +```bash +ansible-playbook -i inventory/hosts.yaml playbooks/compress_logs.yaml \ + -e "year=2025 month=11 start_day=21 end_day=30 remove_after_compress=false" +``` + +### 只压缩不下载 + +```bash +ansible-playbook -i inventory/hosts.yaml playbooks/compress_logs.yaml \ + -e "year=2025 month=12 start_day=1 end_day=31 download_to_local=false" +``` + +### 保留远程压缩包 + +```bash +ansible-playbook -i inventory/hosts.yaml playbooks/compress_logs.yaml \ + -e "year=2025 month=11 start_day=1 end_day=20 remove_local_archives=false" +``` + +## 断点续传 + +使用 rsync 实现断点续传，网络中断后重新执行会自动从中断点继续下载。 + +## 注意事项 + +- 必须显式指定时间参数 +- 确保服务器上安装了 rsync +- 确保有足够的磁盘空间 +- 首次使用建议小范围测试 + +## 测试建议 + +```bash +# check 模式 +ansible-playbook -i inventory/hosts.yaml playbooks/compress_logs.yaml \ + --limit jhlog \ + -e "year=2025 month=11 start_day=21 end_day=21 remove_after_compress=false" \ + --check + +# 单天测试 +ansible-playbook -i inventory/hosts.yaml playbooks/compress_logs.yaml \ + --limit jhlog \ + -e "year=2025 month=11 start_day=21 end_day=21 remove_after_compress=false" +``` diff --git a/inventory/hosts.yaml b/inventory/hosts.yaml new file mode 100644 index 0000000..3d69aeb --- /dev/null +++ b/inventory/hosts.yaml @@ -0,0 +1,26 @@ +logs: + hosts: + 43.226.58.253: + name: jhlog + ansible_user: wyk + ansible_ssh_pass: adjg9815... + + 106.119.167.38: + name: jhlogc + ansible_user: root + ansible_ssh_pass: adjg9815... + + 123.6.147.9: + name: jglog + ansible_user: wyk + ansible_ssh_pass: adjg9815... + + 106.119.166.79: + name: jglogc + ansible_user: root + ansible_ssh_pass: adjg9815... + + 125.122.23.244: + name: mglog + ansible_user: root + ansible_ssh_pass: adjg9815... diff --git a/playbooks/compress_logs.yaml b/playbooks/compress_logs.yaml new file mode 100644 index 0000000..e981ac3 --- /dev/null +++ b/playbooks/compress_logs.yaml @@ -0,0 +1,123 @@ +--- +- name: 压缩和清理服务器日志 + hosts: logs + gather_facts: false + vars: + # 默认值，可以通过命令行参数覆盖 + log_base_path: "/data/roslog/roslog" + # year, month, start_day, end_day 必须通过命令行参数提供 + local_download_path: "/mnt/d/logs" + remove_after_compress: true + download_to_local: true + remove_local_archives: true + + tasks: + - name: 验证必填参数 + fail: + msg: "缺少必填参数！必须提供: year, month, start_day, end_day" + when: year is not defined or month is not defined or start_day is not defined or end_day is not defined + + - name: 显示将要处理的日志路径 + debug: + msg: "处理路径: {{ log_base_path }}/{{ year }}/{{ month }}, 日期范围: {{ start_day }}-{{ end_day }}" + + - name: 确保日志目录存在 + stat: + path: "{{ log_base_path }}/{{ year }}/{{ month }}" + register: log_dir + + - name: 检查日志目录是否存在 + fail: + msg: "日志目录 {{ log_base_path }}/{{ year }}/{{ month }} 不存在" + when: not log_dir.stat.exists + + - name: 压缩指定日期范围的日志文件夹 + shell: | + cd {{ log_base_path }}/{{ year }}/{{ month }} + for day in {{{ start_day }}..{{ end_day }}}; do + if [ -d "$day" ]; then + echo "压缩目录: $day" + tar -czf ${day}.tar.gz $day + else + echo "目录 $day 不存在，跳过" + fi + done + args: + executable: /bin/bash + become: yes + register: compress_result + + - name: 显示压缩结果 + debug: + var: compress_result.stdout_lines + + - name: 列出已生成的压缩文件 + find: + paths: "{{ log_base_path }}/{{ year }}/{{ month }}" + patterns: "*.tar.gz" + register: compressed_files + + - name: 显示压缩文件列表 + debug: + msg: "已生成 {{ compressed_files.files | length }} 个压缩文件" + + - name: 删除已压缩的原始文件夹 + shell: | + cd {{ log_base_path }}/{{ year }}/{{ month }} + for day in {{{ start_day }}..{{ end_day }}}; do + if [ -d "$day" ] && [ -f "${day}.tar.gz" ]; then + echo "删除目录: $day" + rm -rf $day + fi + done + args: + executable: /bin/bash + become: yes + when: remove_after_compress + register: cleanup_result + + - name: 显示清理结果 + debug: + var: cleanup_result.stdout_lines + when: remove_after_compress + + - name: 确保本地下载目录存在 + local_action: + module: file + path: "{{ local_download_path }}/{{ name }}/{{ year }}/{{ month }}" + state: directory + when: download_to_local + become: no + + - name: 下载压缩文件到本地（支持断点续传） + synchronize: + src: "{{ log_base_path }}/{{ year }}/{{ month }}/{{ item.path | basename }}" + dest: "{{ local_download_path }}/{{ name }}/{{ year }}/{{ month }}/" + mode: pull + rsync_opts: + - "--partial" + - "--progress" + - "--compress" + with_items: "{{ compressed_files.files }}" + when: download_to_local + become: yes + + - name: 删除服务器上的压缩文件 + file: + path: "{{ item.path }}" + state: absent + with_items: "{{ compressed_files.files }}" + when: remove_local_archives and download_to_local + become: yes + + - name: 完成信息 + debug: + msg: | + 日志处理完成！ + - 服务器: {{ name }} ({{ inventory_hostname }}) + - 路径: {{ log_base_path }}/{{ year }}/{{ month }} + - 日期范围: {{ start_day }}-{{ end_day }} + - 压缩文件数: {{ compressed_files.files | length }} + {% if download_to_local %} + - 本地下载路径: {{ local_download_path }}/{{ name }}/{{ year }}/{{ month }} + {% endif %} diff --git a/scripts/compress_logs.sh b/scripts/compress_logs.sh new file mode 100644 index 0000000..0104eea --- /dev/null +++ b/scripts/compress_logs.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# 快速执行日志压缩的便捷脚本 + +# 获取脚本所在目录 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# 必填参数 - 没有默认值 +YEAR="" +MONTH="" +START_DAY="" +END_DAY="" +INVENTORY="$PROJECT_ROOT/inventory/hosts.yaml" +PLAYBOOK="$PROJECT_ROOT/playbooks/compress_logs.yaml" + +# 显示帮助信息 +show_help() { + cat << EOF +用法: $0 -y YEAR -m MONTH -s START_DAY -e END_DAY [其他选项] + +必填选项: + -y, --year YEAR 年份 (必填) + -m, --month MONTH 月份 (必填) + -s, --start-day DAY 起始日期 (必填) + -e, --end-day DAY 结束日期 (必填) + +可选选项: + -h, --host HOST 指定单个主机 + --no-rm-origin 不删除原文件夹 + --no-download 不下载到本地 + --no-rm-archive 不删除远程压缩包 + --help 显示此帮助信息 + +示例: + # 压缩2025年11月21-30日的日志 + $0 -y 2025 -m 11 -s 21 -e 30 + + # 压缩2025年12月的所有日志 + $0 -y 2025 -m 12 -s 1 -e 31 + + # 只处理jhlog服务器 + $0 -y 2025 -m 11 -s 21 -e 30 -h jhlog + + # 只压缩不删除原文件 + $0 -y 2025 -m 11 -s 21 -e 30 --no-rm-origin + +注意: + - 年月日参数必须显式指定 + - 下载路径: /mnt/d/logs//// + - 支持断点续传 + +EOF +} + +# 解析命令行参数 +EXTRA_VARS="" +LIMIT="" + +while [[ $# -gt 0 ]]; do + case $1 in + -y|--year) + YEAR="$2" + shift 2 + ;; + -m|--month) + MONTH="$2" + shift 2 + ;; + -s|--start-day) + START_DAY="$2" + shift 2 + ;; + -e|--end-day) + END_DAY="$2" + shift 2 + ;; + -h|--host) + LIMIT="--limit $2" + shift 2 + ;; + --no-rm-origin) + EXTRA_VARS="$EXTRA_VARS remove_after_compress=false" + shift + ;; + --no-download) + EXTRA_VARS="$EXTRA_VARS download_to_local=false" + shift + ;; + --no-rm-archive) + EXTRA_VARS="$EXTRA_VARS remove_local_archives=false" + shift + ;; + --help) + show_help + exit 0 + ;; + *) + echo "未知选项: $1" + show_help + exit 1 + ;; + esac +done + +# 验证必填参数 +if [ -z "$YEAR" ] || [ -z "$MONTH" ] || [ -z "$START_DAY" ] || [ -z "$END_DAY" ]; then + echo "错误: 缺少必填参数！" + echo "" + show_help + exit 1 +fi + +# 构建完整的 extra-vars +EXTRA_VARS="year=$YEAR month=$MONTH start_day=$START_DAY end_day=$END_DAY $EXTRA_VARS" + +# 显示执行信息 +echo "==========================================" +echo "日志压缩任务" +echo "==========================================" +echo "年份: $YEAR" +echo "月份: $MONTH" +echo "日期范围: $START_DAY - $END_DAY" +echo "下载路径: /mnt/d/logs//$YEAR/$MONTH/" +echo "==========================================" +echo "" + +# 执行 ansible-playbook +ansible-playbook -i "$INVENTORY" "$PLAYBOOK" $LIMIT -e "$EXTRA_VARS"