fix auto-update

init-server.py

@@ -12,7 +12,6 @@ from ssh_lib.tasks import (
     setup_ledns_writer,
     setup_loadbalancer,
     upload_config_json,
-    upload_http_host_files,
 )
 from ssh_lib.utils import (
     put,
@@ -93,14 +92,15 @@ def http_host_autoupdate(hostname, user, port):
 
 @cli.command()
 @common_options
-def tile_gen(hostname, user, port):
+@click.option('--cron', is_flag=True, help='Enable cron task')
+def tile_gen(hostname, user, port, cron):
     if not click.confirm(f'Run script on {hostname}?'):
         return
 
     c = get_connection(hostname, user, port)
     prepare_shared(c)
 
-    prepare_tile_gen(c)
+    prepare_tile_gen(c, enable_cron=cron)
 
 
 @cli.command()

modules/http_host/http_host.py

@@ -106,5 +106,4 @@ def debug():
 
 
 if __name__ == '__main__':
-    # print(config.host_config)
     cli()

modules/http_host/http_host_lib/config.py

@@ -1,4 +1,5 @@
 import json
+import subprocess
 from pathlib import Path
 
 
@@ -14,17 +15,17 @@ class Configuration:
     assets_dir = http_host_dir / 'assets'
 
     mnt_dir = Path('/mnt/ofm')
-    ofm_config_dir = Path('/data/ofm/config')
-    deployed_versions_dir = ofm_config_dir / 'deployed_versions'
 
     certs_dir = Path('/data/nginx/certs')
     nginx_confs = Path(__file__).parent / 'nginx_confs'
 
-    try:
+    ofm_config_dir = Path('/data/ofm/config')
-        with open(ofm_config_dir / 'http_host.json') as fp:
+    deployed_versions_dir = ofm_config_dir / 'deployed_versions'
-            host_config = json.load(fp)
+
-    except Exception:
+    ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
-        host_config = {}
+
+    rclone_config = ofm_config_dir / 'rclone.conf'
+    rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip()
 
 
 config = Configuration()

modules/http_host/http_host_lib/nginx.py

@@ -13,8 +13,8 @@ def write_nginx_config():
 
     curl_text_mix = ''
 
-    domain_le = config.host_config['domain_le']
+    domain_le = config.ofm_config['domain_le']
-    domain_ledns = config.host_config['domain_ledns']
+    domain_ledns = config.ofm_config['domain_ledns']
 
     # remove old configs and certs
     for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
@@ -25,7 +25,7 @@ def write_nginx_config():
 
     # processing Round Robin DNS config
     if domain_ledns:
-        if not (config.ofm_config_dir / 'rclone.conf').is_file():
+        if not config.rclone_config.is_file():
             sys.exit('rclone.conf missing')
 
         # download the ledns certificate from bucket using rclone
@@ -64,7 +64,7 @@ def write_nginx_config():
                 '--webroot-path=/data/nginx/acme-challenges',
                 '--noninteractive',
                 '-m',
-                config.host_config['le_email'],
+                config.ofm_config['le_email'],
                 '--agree-tos',
                 '--cert-name=ofm_le',
                 # '--staging',
@@ -125,12 +125,11 @@ def create_location_blocks(*, local, domain):
             area=area, version=version, subdir=subdir, local=local, domain=domain
         )
 
-        if not curl_text:
+        curl_text += (
-            curl_text = (
+            '\ntest with:\n'
-                '\ntest with:\n'
+            f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n'
-                f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n'
+            f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf'
-                f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf'
+        )
-            )
 
     location_str += create_latest_locations(local=local, domain=domain)
 

modules/http_host/http_host_lib/sync.py

@@ -36,7 +36,7 @@ def full_sync(force=False):
     btrfs_downloaded += download_area_version(area='monaco', version='deployed')
 
     # download latest and deployed planet
-    if not config.host_config.get('skip_planet'):
+    if not config.ofm_config.get('skip_planet'):
         btrfs_downloaded += download_area_version(area='planet', version='latest')
         btrfs_downloaded += download_area_version(area='planet', version='deployed')
 

modules/http_host/setup.py

@@ -3,6 +3,7 @@ from setuptools import find_packages, setup
 
 requirements = [
     'click',
+    'pycurl',
     'requests',
 ]
 

modules/tile_gen/cron.d/ofm_tile_gen

@@ -1,15 +1,18 @@
+# Define common variables
+CMD="sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py"
+LOG_DIR=/data/ofm/tile_gen/logs
+
 # every hour, make a monaco run
-10 * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles monaco --upload >> /data/ofm/tile_gen/logs/monaco-make-tiles.log 2>&1
+10 * * * * ofm    $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
 
 # every minute, set monaco to latest
-* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py tile_gen.py set-version monaco >> /data/ofm/tile_gen/logs/monaco-set-version.log 2>&1
+* * * * * ofm     $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1
 
 # every Tuesday morning at 00:10, make a planet run
-10 0 * * 2 ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles planet --upload >> /data/ofm/tile_gen/logs/planet-make-tiles.log 2>&1
+10 0 * * 2 ofm    $CMD make-tiles planet --upload >> $LOG_DIR/planet-make-tiles.log 2>&1
-
-
 
+# every Saturday at 8:00, set planet to latest
+0 9 * * 6 ofm     $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1
 
 # once per minute, create indexes
-* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-indexes >> /data/ofm/tile_gen/logs/make-indexes-cron.log 2>&1
+* * * * * ofm     $CMD make-indexes >> $LOG_DIR/make-indexes-cron.log 2>&1
-

modules/tile_gen/tile_gen_lib/config.py

@@ -21,14 +21,11 @@ class Configuration:
         ofm_config_dir = Path('/data/ofm/config')
     else:
         repo_root = Path(__file__).parent.parent.parent.parent
-
         ofm_config_dir = repo_root / 'config'
 
     ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
-    pprint(ofm_config)
 
     rclone_config = ofm_config_dir / 'rclone.conf'
-
     rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip()
 
 

ssh_lib/tasks.py

@@ -47,7 +47,7 @@ def prepare_venv(c):
     sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh')
 
 
-def prepare_tile_gen(c):
+def prepare_tile_gen(c, *, enable_cron):
     c.sudo('rm -f /etc/cron.d/ofm_tile_gen')
 
     install_planetiler(c)
@@ -76,7 +76,8 @@ def prepare_tile_gen(c):
     c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}')
     c.sudo(f'chown ofm:ofm -R {TILE_GEN_BIN}')
 
-    put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
+    if enable_cron:
+        put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
 
 
 def prepare_http_host(c):