first steps to better scanning of repos

src/repo_registry/storage/sqlite.py

@@ -180,6 +180,14 @@ class RegistryStore:
             )
             """
         )
+        columns = {
+            row["name"]
+            for row in connection.execute("PRAGMA table_info(content_chunks)").fetchall()
+        }
+        if "metadata" not in columns:
+            connection.execute(
+                "ALTER TABLE content_chunks ADD COLUMN metadata TEXT NOT NULL DEFAULT '{}'"
+            )
         connection.execute(
             "CREATE INDEX IF NOT EXISTS idx_content_chunks_repository ON content_chunks(repository_id)"
         )
@@ -1675,8 +1683,8 @@ class RegistryStore:
                 """
                 INSERT INTO content_chunks
                   (repository_id, analysis_run_id, snapshot_id, path, kind,
-                   start_line, end_line, text)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                   start_line, end_line, text, metadata)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
                 [
                     (
@@ -1688,6 +1696,7 @@ class RegistryStore:
                         chunk.start_line,
                         chunk.end_line,
                         chunk.text,
+                        json.dumps(chunk.metadata),
                     )
                     for chunk in chunks
                 ],
@@ -1709,7 +1718,7 @@ class RegistryStore:
             rows = connection.execute(
                 f"""
                 SELECT id, repository_id, analysis_run_id, snapshot_id, path, kind,
-                       start_line, end_line, text
+                       start_line, end_line, text, metadata
                 FROM content_chunks
                 {where}
                 ORDER BY path ASC, start_line ASC, id ASC
@@ -2842,6 +2851,7 @@ class RegistryStore:
             start_line=row["start_line"],
             end_line=row["end_line"],
             text=row["text"],
+            metadata=json.loads(row["metadata"]),
         )
 
     @staticmethod