zxf/log-fetch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

极狐极光日志自动下载处理

Browse Source
This commit is contained in:
luorijun
2025-12-17 16:18:59 +08:00
parent 748d2b26a4
commit 9b39441571
3 changed files with 121 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
inventory/hosts.yaml
View File

@@ -10,7 +10,7 @@ logs:
ansible_user: root ansible_user: root
ansible_ssh_pass: adjg9815... ansible_ssh_pass: adjg9815...
123.6.147.9: 43.226.59.253:
name: jglog name: jglog
ansible_user: wyk ansible_user: wyk
ansible_ssh_pass: adjg9815... ansible_ssh_pass: adjg9815...

234
playbooks/compress_logs.yaml
View File

@@ -1,123 +1,111 @@
--- ---
- name: 压缩和清理服务器日志 - name: 压缩和清理服务器日志
hosts: logs hosts: logs
gather_facts: false gather_facts: false
vars: vars:
# 默认值，可以通过命令行参数覆盖 # year, month, start_day, end_day 通过命令行提供
log_base_path: "/data/roslog/roslog" log_base_path: "/data/roslog/roslog"
# year, month, start_day, end_day 必须通过命令行参数提供 local_download_path: "/mnt/d/logs"
local_download_path: "/mnt/d/logs" remove_after_compress: true
remove_after_compress: true download_to_local: true
download_to_local: true remove_local_archives: true
remove_local_archives: true
tasks:
tasks: - name: 验证必填参数
- name: 验证必填参数 ansible.builtin.fail:
fail: msg: "缺少必填参数！必须提供: year, month, start_day, end_day"
msg: "缺少必填参数！必须提供: year, month, start_day, end_day" when: year is not defined or month is not defined or start_day is not defined or end_day is not defined
when: year is not defined or month is not defined or start_day is not defined or end_day is not defined
- name: 显示将要处理的日志路径
- name: 显示将要处理的日志路径 ansible.builtin.debug:
debug: msg: "处理路径: {{ log_base_path }}/{{ year }}/{{ month }}, 日期范围: {{ start_day }}-{{ end_day }}"
msg: "处理路径: {{ log_base_path }}/{{ year }}/{{ month }}, 日期范围: {{ start_day }}-{{ end_day }}"
- name: 确保日志目录存在
- name: 确保日志目录存在 ansible.builtin.stat:
stat: path: "{{ log_base_path }}/{{ year }}/{{ month }}"
path: "{{ log_base_path }}/{{ year }}/{{ month }}" register: log_dir
register: log_dir
- name: 检查日志目录是否存在
- name: 检查日志目录是否存在 ansible.builtin.fail:
fail: msg: "日志目录 {{ log_base_path }}/{{ year }}/{{ month }} 不存在"
msg: "日志目录 {{ log_base_path }}/{{ year }}/{{ month }} 不存在" when: not log_dir.stat.exists
when: not log_dir.stat.exists
- name: 压缩指定日期范围的日志文件夹
- name: 压缩指定日期范围的日志文件夹 ansible.builtin.shell: |
shell: | cd {{ log_base_path }}/{{ year }}/{{ month }}
cd {{ log_base_path }}/{{ year }}/{{ month }} for day in $(seq -f "%02g" {{ start_day }} {{ end_day }}); do
for day in {{{ start_day }}..{{ end_day }}}; do if [ -d "$day" ]; then
if [ -d "$day" ]; then echo "压缩目录: $day"
echo "压缩目录: $day" tar -czf ${day}.tar.gz $day
tar -czf ${day}.tar.gz $day else
else echo "目录 $day 不存在，跳过"
echo "目录 $day 不存在，跳过" fi
fi done
done args:
args: executable: /bin/bash
executable: /bin/bash become: true
become: yes register: compress_result
register: compress_result
- name: 显示压缩结果
- name: 显示压缩结果 ansible.builtin.debug:
debug: var: compress_result.stdout_lines
var: compress_result.stdout_lines
- name: 列出已生成的压缩文件
- name: 列出已生成的压缩文件 ansible.builtin.find:
find: paths: "{{ log_base_path }}/{{ year }}/{{ month }}"
paths: "{{ log_base_path }}/{{ year }}/{{ month }}" patterns: "*.tar.gz"
patterns: "*.tar.gz" register: compressed_files
register: compressed_files
- name: 删除已压缩的原始文件夹
- name: 显示压缩文件列表 ansible.builtin.shell: |
debug: cd {{ log_base_path }}/{{ year }}/{{ month }}
msg: "已生成 {{ compressed_files.files | length }} 个压缩文件" for day in $(seq -f "%02g" {{ start_day }} {{ end_day }}); do
if [ -d "$day" ] && [ -f "${day}.tar.gz" ]; then
- name: 删除已压缩的原始文件夹 echo "删除原始目录: $day"
shell: | rm -rf $day
cd {{ log_base_path }}/{{ year }}/{{ month }} fi
for day in {{{ start_day }}..{{ end_day }}}; do done
if [ -d "$day" ] && [ -f "${day}.tar.gz" ]; then args:
echo "删除目录: $day" executable: /bin/bash
rm -rf $day become: true
fi when: remove_after_compress
done
args: - name: 确保本地下载目录存在
executable: /bin/bash ansible.builtin.file:
become: yes path: "{{ local_download_path }}/{{ label }}/{{ year }}/{{ month }}"
when: remove_after_compress state: directory
register: cleanup_result delegate_to: localhost
when: download_to_local
- name: 显示清理结果 become: false
debug:
var: cleanup_result.stdout_lines - name: 下载压缩文件到本地
when: remove_after_compress ansible.posix.synchronize:
src: "{{ log_base_path }}/{{ year }}/{{ month }}/{{ item.path | basename }}"
- name: 确保本地下载目录存在 dest: "{{ local_download_path }}/{{ label }}/{{ year }}/{{ month }}/"
local_action: mode: pull
module: file rsync_opts:
path: "{{ local_download_path }}/{{ name }}/{{ year }}/{{ month }}" - "--partial"
state: directory - "--progress"
when: download_to_local with_items: "{{ compressed_files.files }}"
become: no when: download_to_local
become: true
- name: 下载压缩文件到本地（支持断点续传）
synchronize: - name: 删除服务器上的压缩文件
src: "{{ log_base_path }}/{{ year }}/{{ month }}/{{ item.path | basename }}" ansible.builtin.file:
dest: "{{ local_download_path }}/{{ name }}/{{ year }}/{{ month }}/" path: "{{ item.path }}"
mode: pull state: absent
rsync_opts: with_items: "{{ compressed_files.files }}"
- "--partial" when: remove_local_archives and download_to_local
- "--progress" become: true
- "--compress"
with_items: "{{ compressed_files.files }}" - name: 完成信息
when: download_to_local ansible.builtin.debug:
become: yes msg: |
日志处理完成！
- name: 删除服务器上的压缩文件 - 服务器: {{ label }} ({{ inventory_hostname }})
file: - 路径: {{ log_base_path }}/{{ year }}/{{ month }}
path: "{{ item.path }}" - 日期范围: {{ start_day }}-{{ end_day }}
state: absent - 压缩文件数: {{ compressed_files.files | length }}
with_items: "{{ compressed_files.files }}" {% if download_to_local %}
when: remove_local_archives and download_to_local - 本地下载路径: {{ local_download_path }}/{{ label }}/{{ year }}/{{ month }}
become: yes {% endif %}
- name: 完成信息
debug:
msg: |
日志处理完成！
- 服务器: {{ name }} ({{ inventory_hostname }})
- 路径: {{ log_base_path }}/{{ year }}/{{ month }}
- 日期范围: {{ start_day }}-{{ end_day }}
- 压缩文件数: {{ compressed_files.files | length }}
{% if download_to_local %}
- 本地下载路径: {{ local_download_path }}/{{ name }}/{{ year }}/{{ month }}
{% endif %}

24
scripts/compress_logs.sh
View File

@@ -19,17 +19,17 @@ show_help() {
用法: $0 -y YEAR -m MONTH -s START_DAY -e END_DAY [其他选项] 用法: $0 -y YEAR -m MONTH -s START_DAY -e END_DAY [其他选项]
必填选项: 必填选项:
-y, --year YEAR 年份 (必填) -y, --year YEAR 年份
-m, --month MONTH 月份 (必填) -m, --month MONTH 月份
-s, --start-day DAY 起始日期 (必填) -s, --start-day DAY 起始日期
-e, --end-day DAY 结束日期 (必填) -e, --end-day DAY 结束日期
可选选项: 可选选项:
-h, --host HOST 指定单个主机 -h, --host HOST 指定单个主机
--no-rm-origin 不删除原文件夹 --no-rm-origin 不删除原文件夹
--no-download 不下载到本地 --no-download 不下载到本地
--no-rm-archive 不删除远程压缩包 --no-rm-archive 不删除远程压缩包
--help 显示此帮助信息 --help 显示此帮助信息
示例: 示例:
# 压缩2025年11月21-30日的日志 # 压缩2025年11月21-30日的日志
@@ -43,12 +43,6 @@ show_help() {
# 只压缩不删除原文件 # 只压缩不删除原文件
$0 -y 2025 -m 11 -s 21 -e 30 --no-rm-origin $0 -y 2025 -m 11 -s 21 -e 30 --no-rm-origin
注意:
- 年月日参数必须显式指定
- 下载路径: /mnt/d/logs/<host_name>/<year>/<month>/
- 支持断点续传
EOF EOF
} }