Tighter schema, and handle failed downloads

Makefile

@@ -1,8 +1,8 @@
-.PHONY: all install_requirements download combine
+.PHONY: all install_requirements download combine validate list
 
 PYTHON := $(shell which python3 2>/dev/null || which python)
 
-all: install_requirements download combine list
+all: install_requirements download combine validate list
 
 install_requirements:
 	$(PYTHON) -m pip install -r lib/requirements.txt

Schema.json

@@ -22,7 +22,7 @@
             "type": "integer",
             "minimum": 1,
             "maximum": 4,
-            "description": "The type of the application (1=container, 2=swarm stack, 3=compose stack, 4=kubernetes)."
+            "description": "The type of the application (1=container, 2=swarm stack, 3=compose stack, 4=compose edge stack)."
           },
           "title": {
             "type": "string",
@@ -44,8 +44,8 @@
           },
           "platform": {
             "type": "string",
-            "minLength": 1,
-            "description": "The target platform of the application (e.g., 'linux', 'windows')."
+            "enum": ["linux", "windows"],
+            "description": "The target platform of the application; omit for multi-platform."
           },
           "logo": {
             "type": "string",
@@ -89,6 +89,7 @@
                   "description": "Whether the volume is read-only."
                 }
               },
+              "additionalProperties": false,
               "required": ["container"]
             },
             "description": "An array of volume mappings for the application."
@@ -128,11 +129,13 @@
                       "value": { "type": "string" },
                       "default": { "type": "boolean" }
                     },
+                    "additionalProperties": false,
                     "required": ["text", "value"]
                   },
                   "description": "Dropdown options for the environment variable."
                 }
               },
+              "additionalProperties": false,
               "required": ["name"]
             },
             "description": "Environment variables for the template."
@@ -151,6 +154,7 @@
                   "minLength": 1
                 }
               },
+              "additionalProperties": false,
               "required": ["name", "value"]
             },
             "description": "Docker labels for the template."
@@ -169,6 +173,7 @@
                 "description": "Path to the stack file within the repository."
               }
             },
+            "additionalProperties": false,
             "required": ["url", "stackfile"],
             "description": "Repository details for stack templates (types 2 and 3)."
           },
@@ -211,8 +216,13 @@
           "stackFile": {
             "type": "string",
             "description": "Inline stack file content. When present, Portainer uses this directly instead of cloning the repository."
+          },
+          "maintainer": {
+            "type": "string",
+            "description": "Maintainer of the template (metadata added by this repo; ignored by Portainer)."
           }
         },
+        "additionalProperties": false,
         "required": ["id", "type", "title", "description"],
         "allOf": [
           {
@@ -226,10 +236,15 @@
           {
             "if": { "properties": { "type": { "const": 3 } } },
             "then": { "required": ["repository"] }
+          },
+          {
+            "if": { "properties": { "type": { "const": 4 } } },
+            "then": { "anyOf": [{ "required": ["stackFile"] }, { "required": ["repository"] }] }
           }
         ]
       }
     }
   },
+  "additionalProperties": false,
   "required": ["version", "templates"]
 }

lib/combine.py

@@ -1,11 +1,13 @@
 import json
 import os
+import re
 import string
+import sys
 
 BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
 SOURCES_DIR = os.path.join(BASE_DIR, 'sources')
 
-TYPE_LABELS = {1: 'container', 2: 'swarm', 3: 'stack', 4: 'kubernetes'}
+TYPE_LABELS = {1: 'container', 2: 'swarm', 3: 'stack', 4: 'edge'}
 
 reset_color = "\033[0m"
 def rgb(r, g, b):
@@ -15,6 +17,16 @@ def normalize_string(original, lowercase=True):
   normalized = original.translate(str.maketrans('', '', string.punctuation)).replace(' ', '')
   return normalized.lower() if lowercase else normalized.capitalize()
 
+CATEGORY_ACRONYMS = {'ai', 'vpn', 'iot', 'nas', 'dns', 'ci', 'cd', 'tv', 'os', 'ip', 'sql',
+                     'api', 'cms', 'ftp', 'rss', 'cctv', 'llm', 'osint', 'pdf', '3d'}
+
+def normalize_category(c):
+  """Split a category into words and title-case it for display"""
+  words = [w for w in re.split(r'[^0-9a-zA-Z]+', c) if w]
+  label = ' '.join(w.upper() if w.lower() in CATEGORY_ACRONYMS else w.capitalize()
+                   for w in words)
+  return 'edge' if label == 'Edge' else label
+
 def template_score(t):
   """Score a template's completeness. Higher is more complete."""
   score = len(t)  # number of top-level keys
@@ -59,6 +71,14 @@ def normalize_template_fields(templates):
       merged = list(dict.fromkeys(existing + t.pop('category')))
       t['categories'] = merged
 
+    # Convert legacy v2 edge templates (type 4 with a stackfile URL) to compose stacks
+    if t.get('type') == 4 and isinstance(t.get('stackfile'), str):
+      m = re.match(r'https://raw\.githubusercontent\.com/([^/]+/[^/]+)/[^/]+/(.+)', t.pop('stackfile'))
+      if m:
+        t['type'] = 3
+        t['repository'] = {'url': f'https://github.com/{m[1]}', 'stackfile': m[2]}
+        t.setdefault('categories', ['edge'])
+
     # Fix env vars
     if 'env' in t:
       cleaned_env = []
@@ -72,9 +92,18 @@ def normalize_template_fields(templates):
           env.setdefault('preset', True)
         elif 'set' in env:
           del env['set']
-        cleaned_env.append(env)
+        cleaned_env.append({k: v for k, v in env.items() if k in VALID_ENV_KEYS})
       t['env'] = cleaned_env
 
+    # Drop malformed leading ':' from port mappings (e.g. ':80/tcp')
+    if 'ports' in t:
+      t['ports'] = [p.lstrip(':') for p in t['ports']]
+
+    if isinstance(t.get('maintainer'), str):
+      t['maintainer'] = t['maintainer'].strip()
+
+    t.pop('devices', None)  # not part of the Portainer template spec
+
     # Fix volume 'read_only' -> 'readonly' (and coerce to bool)
     for vol in t.get('volumes', []):
       if 'read_only' in vol:
@@ -86,10 +115,14 @@ def is_valid_template(t):
   if not isinstance(t.get('title'), str) or not isinstance(t.get('description'), str):
     return False
   tmpl_type = t.get('type', 1)
+  if not isinstance(tmpl_type, int):
+    return False
   if tmpl_type == 1 and 'image' not in t:
     return False
   if tmpl_type in (2, 3) and 'repository' not in t:
     return False
+  if tmpl_type == 4 and 'repository' not in t and 'stackFile' not in t:
+    return False
   return True
 
 def deduplicate_and_normalize(templates):
@@ -117,9 +150,12 @@ def deduplicate_and_normalize(templates):
       best[key] = t
   result = []
   for t in best.values():
-    t['categories'] = list(dict.fromkeys(
-      normalize_string(c, lowercase=False) for c in t.get('categories', [])
-    ))
+    cats = {}
+    for c in t.get('categories', []):
+      label = normalize_category(c)
+      if label:
+        cats.setdefault(label.lower(), label)
+    t['categories'] = list(cats.values())
     result.append(t)
   return result
 
@@ -157,6 +193,16 @@ if __name__ == '__main__':
   templates.sort(key=lambda t: t['title'].lower())
   for i, t in enumerate(templates, start=1):
     t['id'] = i
+  out_path = os.path.join(BASE_DIR, 'templates.json')
+  try:
+    with open(out_path) as f:
+      previous = len(json.load(f)['templates'])
+  except (OSError, ValueError, KeyError):
+    previous = 0
+  # A failed source download must not silently shrink the published list
+  if len(templates) < previous * 0.9 and not os.environ.get('ALLOW_SHRINK'):
+    sys.exit(f'Refusing to write: template count fell from {previous} to {len(templates)}. '
+             'Set ALLOW_SHRINK=1 if this is intentional.')
   output = {'version': '3', 'templates': templates}
-  with open(os.path.join(BASE_DIR, 'templates.json'), 'w') as f:
+  with open(out_path, 'w') as f:
     json.dump(output, f, indent=2, sort_keys=False)

lib/download.py

@@ -8,40 +8,26 @@ dir = os.path.dirname(os.path.abspath(__file__))
 destination_dir = os.path.join(dir, '../sources/external')
 sources_list = os.path.join(dir, '../sources.csv')
 
-# Downloads the file from a given URL, to the local destination
+# Downloads the templates file from a given URL, to the local destination
 def download(url: str, filename: str, maintainer: str):
     file_path = os.path.join(destination_dir, filename)
     print('Downloading', url)
-    r = requests.get(url, stream=True)
-    if r.ok:
-        print('saving to', os.path.abspath(file_path))
-        with open(file_path, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=1024 * 8):
-                if chunk:
-                    f.write(chunk)
-                    f.flush()
-                    os.fsync(f.fileno())
+    try:
+        r = requests.get(url, timeout=30)
+        r.raise_for_status()
+        sourceJson = r.json()
+    except (requests.RequestException, ValueError) as err:
+        print(f'Skipping source due to an error: {url}')
+        print(f'Error msg: {err}')
+        return
 
-        sourceJson = {}
-        with open(file_path) as f:
-            try:
-                sourceJson = json.load(f)
-                # Add maintainer field to each template
-                for t in sourceJson.get('templates', []):
-                    t['maintainer'] = maintainer
+    # Add maintainer field to each template
+    for t in sourceJson.get('templates', []):
+        t['maintainer'] = maintainer
 
-            except json.decoder.JSONDecodeError as err:
-                print(f'Skipping one of the sources due to an error: {f.name}')
-                print(f'Error msg: {err.msg}')
-
-        if not sourceJson:
-           return
-
-        with open(file_path, 'w') as f:
-            json.dump(sourceJson, f, indent=2, sort_keys=False)
-
-    else:  # HTTP status code 4XX/5XX
-        print('Download failed: status code {}\n{}'.format(r.status_code, r.text))
+    print('saving to', os.path.abspath(file_path))
+    with open(file_path, 'w') as f:
+        json.dump(sourceJson, f, indent=2, sort_keys=False)
 
 # Gets list of URLs to download from CSV file
 def get_source_list():
@@ -49,8 +35,8 @@ def get_source_list():
   with open(sources_list, mode='r') as file:
       csvFile = csv.reader(file)
       for lines in csvFile:
-        if len(lines) > 1 and lines[1].strip():
-          sources.append(lines)
+        if len(lines) > 2 and lines[1].strip():
+          sources.append([col.strip() for col in lines])
   return sources
 
 # Create destination folder if not yet present

lib/list.py

@@ -20,7 +20,13 @@ def load_csv_file(file_path):
 
 def slugify(title: str):
     baseUrl = 'https://portainer-templates.as93.net'
-    return f'{baseUrl}/{re.sub(r"[^a-zA-Z ]", "", title.lower()).replace(" ", "-")}'
+    slug = re.sub(r'\s+', '-', re.sub(r'[^a-z0-9 ]', '', title.lower()).strip())
+    return f'{baseUrl}/{slug}'
+
+def clean_text(text):
+    """Remove quotes/brackets and duplicate whitespace for html/md render"""
+    text = (text or '').replace('"', '”').replace("'", '’').replace('<', '').replace('>', '')
+    return re.sub(r'\s+', ' ', text).strip()
 
 def generate_app_list():
   templates = load_json_file(templates_path)['templates']
@@ -30,7 +36,7 @@ def generate_app_list():
       name = template['title'].title()
       maintainer = template.get('maintainer')
       maintainer_md_link = f" -- ([Report issues]({maintainer}))" if maintainer else ''
-      description = re.sub('[^0-9a-zA-Z]+', ' ', (template['description'] or ''))
+      description = clean_text(template['description'])
       if 'logo' in template and template['logo']:
           logo = f"<img title=\"{description}\" src='{template['logo']}' width='26' height='26' /> "
       else:

sources.csv

@@ -1,12 +1,10 @@
 dnburgess_templates, https://raw.githubusercontent.com/dnburgess/self-hosted-template/master/template.json, https://github.com/dnburgess/self-hosted-template/
 qballjos_templates, https://raw.githubusercontent.com/Qballjos/portainer_templates/master/Template/template.json, https://github.com/Qballjos/portainer_templates/
 selfhostedpro_templates, https://raw.githubusercontent.com/SelfhostedPro/selfhosted_templates/portainer-2.0/Template/template.json, https://github.com/SelfhostedPro/selfhosted_templates/
-technorabilia_templates, https://raw.githubusercontent.com/technorabilia/portainer-templates/main/lsio/templates/templates-2.0.json, https://github.com/technorabilia/portainer-templates/
+technorabilia_templates, https://raw.githubusercontent.com/technorabilia/portainer-templates/main/lsio/templates/templates.json, https://github.com/technorabilia/portainer-templates/
 mikestraney_templates, https://raw.githubusercontent.com/mikestraney/portainer-templates/master/templates.json, https://github.com/mikestraney/portainer-templates/
 xneo1_templates, https://raw.githubusercontent.com/xneo1/portainer_templates/master/Template/template.json, https://github.com/xneo1/portainer_templates/
 novaspirit_templates, https://raw.githubusercontent.com/novaspirit/pi-hosted/master/pi-hosted_template/template/portainer-v2.json, https://github.com/novaspirit/pi-hosted/
-donpablonow_templates, https://raw.githubusercontent.com/donpablonow/awesome-saas/master/Template/portainer-v2.json, https://github.com/donpablonow/awesome-saas/
 mediadepot_templates, https://raw.githubusercontent.com/mediadepot/templates/master/portainer.json, https://github.com/mediadepot/templates/
-mycroftwilde_templates, https://raw.githubusercontent.com/mycroftwilde/portainer_templates/master/Template/template.json, https://github.com/mycroftwilde/portainer_templates/
 shmolf_templates, https://raw.githubusercontent.com/shmolf/portainer-templates/main/templates-2.0.json, https://github.com/shmolf/portainer-templates/
-portainer_templates, https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json, https://github.com/portainer/templates/
+portainer_templates, https://raw.githubusercontent.com/portainer/templates/v3/templates.json, https://github.com/portainer/templates/