From f9649c50fc1931d06d8c61e9c5d15a7d3fe0f54d Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Mon, 4 Dec 2023 22:56:27 +0100 Subject: [PATCH] work --- .gitignore | 1 + .ruff.toml | 11 ++++--- init-server.py | 29 +++++++++-------- {openfreemaps => lib}/__init__.py | 0 {openfreemaps => lib}/config.py | 0 lib/dns.py | 29 +++++++++++++++++ openfreemaps/system.py => lib/kernel.py | 19 ++--------- {openfreemaps => lib}/nginx.py | 15 ++++----- {openfreemaps => lib}/pkg_base.py | 9 +++++- {openfreemaps => lib}/planetiler.py | 10 +++--- {openfreemaps => lib}/utils.py | 42 +++++++++++++++++++++++++ setup.py | 4 +-- templates/nginx/cloudflare.conf | 30 ++++++++++++++++++ 13 files changed, 150 insertions(+), 49 deletions(-) rename {openfreemaps => lib}/__init__.py (100%) rename {openfreemaps => lib}/config.py (100%) create mode 100644 lib/dns.py rename openfreemaps/system.py => lib/kernel.py (55%) rename {openfreemaps => lib}/nginx.py (79%) rename {openfreemaps => lib}/pkg_base.py (74%) rename {openfreemaps => lib}/planetiler.py (68%) rename {openfreemaps => lib}/utils.py (68%) create mode 100644 templates/nginx/cloudflare.conf diff --git a/.gitignore b/.gitignore index 317070a..90f3477 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ .DS_Store /venv /.idea +/temp diff --git a/.ruff.toml b/.ruff.toml index f65f4b7..325dbdd 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,6 +1,6 @@ target-version = "py310" line-length = 100 - +extend-exclude = ["temp"] select = [ "E", # pycodestyle errors @@ -19,24 +19,25 @@ select = [ ] ignore = [ + 'A003', 'E501', 'E711', 'E712', 'E741', - 'A003', + 'F401', + 'F841', 'PT004', - 'SIM108', 'SIM102', 'SIM105', + 'SIM108', 'SIM115', - 'F841', ] [format] quote-style = "single" [isort] -known-first-party = ["openfreemaps"] +known-first-party = ["lib"] lines-after-imports = 2 [flake8-comprehensions] diff --git a/init-server.py b/init-server.py index af94ab0..c990c5f 100755 --- a/init-server.py +++ b/init-server.py @@ -2,28 +2,31 @@ from fabric import Connection -from openfreemaps.nginx import certbot, nginx -from openfreemaps.pkg_base import pkg_base, pkg_clean, pkg_upgrade -from openfreemaps.planetiler import install_planetiler -from openfreemaps.system import set_cpu_governor, setup_kernel_settings, setup_time +from lib.nginx import certbot, nginx +from lib.pkg_base import pkg_base, pkg_clean, pkg_upgrade +from lib.planetiler import install_planetiler +from lib.system1 import set_cpu_governor, setup_kernel_settings, setup_time +from lib.utils import add_user def prepare_server(c): - pkg_upgrade(c) - pkg_clean(c) - pkg_base(c) + add_user(c, 'ofm') - setup_time(c) - setup_kernel_settings(c) - set_cpu_governor(c) + # pkg_upgrade(c) + # pkg_clean(c) + # pkg_base(c) - nginx(c) - certbot(c) + # setup_time(c) + # setup_kernel_settings(c) + # set_cpu_governor(c) + + # nginx(c) + # certbot(c) install_planetiler(c) -c = Connection(host='map128', port=22) +c = Connection(host='ofm-o-ca-1', port=22, user='ubuntu') prepare_server(c) # reboot(c) diff --git a/openfreemaps/__init__.py b/lib/__init__.py similarity index 100% rename from openfreemaps/__init__.py rename to lib/__init__.py diff --git a/openfreemaps/config.py b/lib/config.py similarity index 100% rename from openfreemaps/config.py rename to lib/config.py diff --git a/lib/dns.py b/lib/dns.py new file mode 100644 index 0000000..a579f1c --- /dev/null +++ b/lib/dns.py @@ -0,0 +1,29 @@ +import time + +from lib.utils import apt_get_purge, exists, put_str + + +def setup_dns(c): + if exists(c, '/etc/network/interfaces'): + c.sudo("sed -i '/dns-nameservers/d' /etc/network/interfaces") + + apt_get_purge(c, 'resolvconf') + c.sudo('rm -rf /etc/resolvconf') + + c.sudo('systemctl stop systemd-resolved') + c.sudo('systemctl disable systemd-resolved') + + print('chattr -i') + c.sudo('chattr -i /etc/resolv.conf', warn=True) + c.sudo('rm -f /etc/resolv.conf') + put_str( + c, + '/etc/resolv.conf', + 'nameserver 1.1.1.1\nnameserver 1.0.0.1\nnameserver 2606:4700:4700::1111\nnameserver 2606:4700:4700::1001', + ) + time.sleep(1) + print('chattr +i') + c.sudo('chattr +i /etc/resolv.conf') + + apt_get_purge(c, 'bind9*') + c.sudo('rm -rf /var/cache/bind') diff --git a/openfreemaps/system.py b/lib/kernel.py similarity index 55% rename from openfreemaps/system.py rename to lib/kernel.py index 6b31703..79ed393 100644 --- a/openfreemaps/system.py +++ b/lib/kernel.py @@ -1,28 +1,15 @@ -from openfreemaps.config import templates -from openfreemaps.utils import ( - apt_get_install, - apt_get_purge, - put, - put_str, -) - - -def setup_time(c): - apt_get_install(c, 'dbus') - - c.sudo('timedatectl set-local-rtc 0') - c.sudo('timedatectl set-ntp 1') - c.sudo('timedatectl set-timezone UTC') +from lib.config import templates +from lib.utils import apt_get_install, apt_get_purge, put, put_str def setup_kernel_settings(c): put(c, f'{templates}/sysctl/60-optim.conf', '/etc/sysctl.d/') - def set_cpu_governor(c): apt_get_install(c, 'cpufrequtils') apt_get_purge(c, 'linux-tools-*') + # c.run('systemctl disable ondemand') # not working on 22 put_str( c, diff --git a/openfreemaps/nginx.py b/lib/nginx.py similarity index 79% rename from openfreemaps/nginx.py rename to lib/nginx.py index ca87b0c..13952b0 100644 --- a/openfreemaps/nginx.py +++ b/lib/nginx.py @@ -1,5 +1,5 @@ -from openfreemaps.config import templates -from openfreemaps.utils import ( +from lib.config import templates +from lib.utils import ( apt_get_install, apt_get_purge, apt_get_update, @@ -45,6 +45,7 @@ def nginx(c): put(c, f'{templates}/nginx/nginx.conf', '/etc/nginx/') put(c, f'{templates}/nginx/default_disable.conf', '/data/nginx/sites') + put(c, f'{templates}/nginx/cloudflare.conf', '/data/nginx/config') c.sudo('service nginx restart') @@ -52,10 +53,10 @@ def nginx(c): def certbot(c): # https://certbot.eff.org/lets-encrypt/ubuntubionic-nginx apt_get_install(c, 'snapd') - c.run('snap install core', warn=True) - c.run('snap refresh core', warn=True) + c.sudo('snap install core', warn=True) + c.sudo('snap refresh core', warn=True) apt_get_purge(c, 'certbot') - c.run('snap install --classic certbot', warn=True) - c.run('snap set certbot trust-plugin-with-root=ok') - c.run('snap install certbot-dns-cloudflare') + c.sudo('snap install --classic certbot', warn=True) + c.sudo('snap set certbot trust-plugin-with-root=ok') + c.sudo('snap install certbot-dns-cloudflare') diff --git a/openfreemaps/pkg_base.py b/lib/pkg_base.py similarity index 74% rename from openfreemaps/pkg_base.py rename to lib/pkg_base.py index 32e3644..b907a22 100644 --- a/openfreemaps/pkg_base.py +++ b/lib/pkg_base.py @@ -1,4 +1,4 @@ -from openfreemaps.utils import ( +from lib.utils import ( apt_get_autoremove, apt_get_install, apt_get_purge, @@ -20,11 +20,18 @@ def pkg_clean(c): 'ufw', 'nftables', 'firewalld', + 'iptables-persistent', + # bloat + 'ntfs-3g', + 'popularity-contest', + 'landscape*', + 'ubuntu-advantage-tools', ] apt_get_purge(c, ' '.join(clean_list)) apt_get_autoremove(c) sudo_cmd(c, 'dpkg --list | grep "^rc" | cut -d " " -f 3 | xargs -r dpkg --purge') + c.sudo('iptables -L') def pkg_base(c): diff --git a/openfreemaps/planetiler.py b/lib/planetiler.py similarity index 68% rename from openfreemaps/planetiler.py rename to lib/planetiler.py index cce7f38..0737bc1 100644 --- a/openfreemaps/planetiler.py +++ b/lib/planetiler.py @@ -1,5 +1,5 @@ -from openfreemaps.config import templates -from openfreemaps.utils import apt_get_install, apt_get_update, put +from lib.config import templates +from lib.utils import apt_get_install, apt_get_update, put PLANETILER_VERSION = '0.7.0' @@ -12,12 +12,12 @@ def install_planetiler(c): apt_get_update(c) apt_get_install(c, 'openjdk-17-jdk') - c.run('mkdir -p /data/planetiler/bin') + c.sudo('mkdir -p /data/planetiler/bin') - c.run( + c.sudo( f'wget -q https://github.com/onthegomap/planetiler/releases/download/v{PLANETILER_VERSION}/planetiler.jar ' f'-O {PLANETILER_PATH}', ) - c.run(f'java -jar {PLANETILER_PATH} --help') + c.sudo(f'java -jar {PLANETILER_PATH} --help') put(c, templates / 'planetiler' / 'run_planet.sh', PLANETILER_DIR, permissions='755') diff --git a/openfreemaps/utils.py b/lib/utils.py similarity index 68% rename from openfreemaps/utils.py rename to lib/utils.py index da7511b..6bdd7eb 100644 --- a/openfreemaps/utils.py +++ b/lib/utils.py @@ -101,3 +101,45 @@ def apt_get_autoremove(c): def get_username(c): return c.run('whoami').stdout.strip() + + +def add_user(c, username, passwd=None): + # ssh-key login only + c.sudo(f'adduser --disabled-password --gecos "" {username}', warn=True) + if passwd: + c.sudo(f'echo "{username}:{passwd}" | chpasswd') + + +def remove_user(c, username): + c.sudo(f'userdel -r {username}', warn=True) + c.sudo(f'rm -rf /home/{username}') + + +def enable_sudo(c, username): + c.sudo(f'usermod -aG sudo {username}') + + +def ssh_copy_id(c, username, key_file_path): + with open(key_file_path) as fp: + public_key_str = fp.read() + + if username == 'root': + home_dir = '/root' + else: + home_dir = f'/home/{username}' + + ssh_dir = f'{home_dir}/.ssh' + + c.sudo(f'mkdir -p {ssh_dir}') + c.sudo(f'chown {username}:{username} {ssh_dir}') + + put_str(c, f'{ssh_dir}/authorized_keys', public_key_str) + set_permission(c, f'{ssh_dir}/authorized_keys', '400', username, username) + + +def setup_time(c): + apt_get_install(c, 'dbus') + + c.sudo('timedatectl set-local-rtc 0') + c.sudo('timedatectl set-ntp 1') + c.sudo('timedatectl set-timezone UTC') diff --git a/setup.py b/setup.py index 942c55e..cfa12ab 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,6 @@ requirements = ['fabric', 'ruff'] setup( python_requires='>=3.10', install_requires=requirements, - name='openfreemaps', - packages=['openfreemaps'], + name='lib', + packages=['lib'], ) diff --git a/templates/nginx/cloudflare.conf b/templates/nginx/cloudflare.conf new file mode 100644 index 0000000..b04720a --- /dev/null +++ b/templates/nginx/cloudflare.conf @@ -0,0 +1,30 @@ +# https://www.cloudflare.com/ips/ + +set_real_ip_from 103.21.244.0/22; +set_real_ip_from 103.22.200.0/22; +set_real_ip_from 103.31.4.0/22; +set_real_ip_from 104.16.0.0/13; +set_real_ip_from 104.24.0.0/14; +set_real_ip_from 108.162.192.0/18; +set_real_ip_from 131.0.72.0/22; +set_real_ip_from 141.101.64.0/18; +set_real_ip_from 162.158.0.0/15; +set_real_ip_from 172.64.0.0/13; +set_real_ip_from 173.245.48.0/20; +set_real_ip_from 188.114.96.0/20; +set_real_ip_from 190.93.240.0/20; +set_real_ip_from 197.234.240.0/22; +set_real_ip_from 198.41.128.0/17; + +set_real_ip_from 2400:cb00::/32; +set_real_ip_from 2405:8100::/32; +set_real_ip_from 2405:b500::/32; +set_real_ip_from 2606:4700::/32; +set_real_ip_from 2803:f800::/32; +set_real_ip_from 2a06:98c0::/29; +set_real_ip_from 2c0f:f248::/32; + +# use any of the following two +real_ip_header CF-Connecting-IP; +#real_ip_header X-Forwarded-For; +