[#131] Support ContentLayout v3 (Unity 6.7) with v2 backward compatibility

Analyzer/Properties/Resources.resx

@@ -247,6 +247,12 @@
   <data name="ContentLayoutLoadableObjects" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>..\Resources\ContentLayoutLoadableObjects.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
   </data>
+  <data name="ContentLayoutLoadableObjectsV2" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutLoadableObjectsV2.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutLoadableObjectsView" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutLoadableObjectsView.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
   <data name="ContentLayoutLoadableScenes" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>..\Resources\ContentLayoutLoadableScenes.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
   </data>

Analyzer/Resources/ContentLayoutLoadableDependencies.sql

@@ -2,5 +2,5 @@ CREATE TABLE IF NOT EXISTS content_layout_loadable_dependencies
 (
     -- The loadable objects each content file references.
     serialized_file_index INTEGER,   -- references content_layout_serialized_files.file_index
-    object_id_hash TEXT              -- references content_layout_loadable_objects.object_id_hash
+    loadable_index INTEGER           -- references content_layout_loadable_objects.loadable_index
 );

Analyzer/Resources/ContentLayoutLoadableObjects.sql

@@ -2,13 +2,11 @@ CREATE TABLE IF NOT EXISTS content_layout_loadable_objects
 (
     -- The objects that can be loaded on demand, identified independently of the content file that
     -- holds them, plus where each came from in the source project.
-    object_id_hash TEXT,             -- hash of GUID, LFID and identifier_type
+    loadable_index INTEGER,          -- the json array index; how the other content_layout tables name a loadable
     guid TEXT,                       -- AssetDatabase GUID of the source asset
-    asset_path TEXT,
-    lfid INTEGER,                    -- local file id of the object in the source asset
+    lfid INTEGER,                    -- local file id of the object in its output content file
     identifier_type INTEGER,
     serialized_file_index INTEGER,   -- content_layout_serialized_files.file_index; NULL if dropped from the build
-    output_lfid INTEGER,             -- local file id of the object in its output content file
-    is_root_asset INTEGER,
-    PRIMARY KEY (object_id_hash)
+    is_root_asset INTEGER,           -- 1-based position in the layout's RootAssets; 0 when not a root asset
+    PRIMARY KEY (loadable_index)
 );

Analyzer/Resources/ContentLayoutLoadableObjectsV2.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS content_layout_loadable_objects
+(
+    -- The objects that can be loaded on demand, identified independently of the content file that
+    -- holds them, plus where each came from in the source project. This is the variant created when
+    -- a version 2 layout (Unity 6.6) is imported; it adds source columns that newer layout versions
+    -- no longer record.
+    loadable_index INTEGER,          -- the json array index; how the other content_layout tables name a loadable
+    guid TEXT,                       -- AssetDatabase GUID of the source asset
+    lfid INTEGER,                    -- local file id of the object in its output content file
+    identifier_type INTEGER,
+    serialized_file_index INTEGER,   -- content_layout_serialized_files.file_index; NULL if dropped from the build
+    is_root_asset INTEGER,           -- 1-based position in the layout's RootAssets; 0 when not a root asset
+    asset_path TEXT,                 -- v2 layouts only: path of the source asset
+    source_lfid INTEGER,             -- v2 layouts only: local file id of the object in the source asset
+    PRIMARY KEY (loadable_index)
+);

Analyzer/Resources/ContentLayoutLoadableObjectsView.sql

@@ -0,0 +1,11 @@
+CREATE VIEW IF NOT EXISTS content_layout_loadable_objects_view AS
+-- Loadables resolved to their analyzed object. The object columns are NULL in a layout-only
+-- database. Selects the loadable table's columns as-is, so the v2-only columns (asset_path,
+-- source_lfid) appear when the imported layout was version 2.
+SELECT l.*,
+       f.filename,
+       o.id AS object, t.name AS type, o.name, o.size
+FROM content_layout_loadable_objects l
+LEFT JOIN content_layout_serialized_files_view f ON f.file_index = l.serialized_file_index
+LEFT JOIN objects o ON o.serialized_file = f.serialized_file AND o.object_id = l.lfid
+LEFT JOIN types t ON t.id = o.type;

Analyzer/Resources/ContentLayoutSerializedFiles.sql

@@ -3,9 +3,9 @@ CREATE TABLE IF NOT EXISTS content_layout_serialized_files
     -- The content files (.cf) the ContentDirectory build produced, one row per entry in the
     -- layout's SerializedFiles array.
     file_index INTEGER,         -- the json array index; how the other content_layout tables name a file
-    cfid TEXT,                  -- symbolic .cfid reference string (for built-ins: the built-in path)
+    stable_id TEXT,             -- identity hash used to reference this file (for built-ins: the built-in path)
     is_builtin INTEGER,
-    content_hash TEXT,          -- NULL for built-ins; the filename is content_hash || '.cf'
+    artifact_index INTEGER,     -- content_layout_binary_artifacts.artifact_index; NULL for built-ins
     serialized_file INTEGER,    -- serialized_files.id; NULL for built-ins and for a layout-only analyze
     PRIMARY KEY (file_index)
 );

Analyzer/Resources/ContentLayoutViews.sql

@@ -1,39 +1,31 @@
 CREATE VIEW IF NOT EXISTS content_layout_serialized_files_view AS
--- One row per layout content file with its derived filename, artifact size and core-table link.
--- Built-in entries have no file on disk, so their path (cfid) is shown as the filename.
-SELECT f.file_index, f.cfid, f.is_builtin,
-       CASE WHEN f.is_builtin = 1 THEN f.cfid ELSE f.content_hash || '.cf' END AS filename,
+-- One row per layout content file with its content hash, derived filename, artifact size and
+-- core-table link. Built-in entries have no file on disk, so their path (stable_id) is shown as
+-- the filename.
+SELECT f.file_index, f.stable_id, f.is_builtin, f.artifact_index,
+       CASE WHEN f.is_builtin = 1 THEN f.stable_id ELSE ba.content_hash || '.cf' END AS filename,
+       ba.content_hash,
        ba.size,
        f.serialized_file,
        sf.archive
 FROM content_layout_serialized_files f
-LEFT JOIN content_layout_binary_artifacts ba ON ba.content_hash = f.content_hash AND ba.category = 'contentfile'
+LEFT JOIN content_layout_binary_artifacts ba ON ba.artifact_index = f.artifact_index
 LEFT JOIN serialized_files sf ON sf.id = f.serialized_file;
 
 CREATE VIEW IF NOT EXISTS content_layout_source_assets_view AS
 -- Source asset to the content file(s) it was built into.
-SELECT s.asset_path, f.file_index, f.content_hash || '.cf' AS filename, f.serialized_file
+SELECT s.asset_path, f.file_index, f.filename, f.serialized_file
 FROM content_layout_source_assets s
-INNER JOIN content_layout_serialized_files f ON f.file_index = s.serialized_file_index;
+INNER JOIN content_layout_serialized_files_view f ON f.file_index = s.serialized_file_index;
 
 CREATE VIEW IF NOT EXISTS content_layout_serialized_file_dependencies_view AS
 -- File-to-file dependency edges with filenames resolved on both sides.
 SELECT d.serialized_file_index, src.filename, d.position,
-       d.dependency_index, dep.filename AS dependency_filename, dep.cfid AS dependency_cfid
+       d.dependency_index, dep.filename AS dependency_filename, dep.stable_id AS dependency_stable_id
 FROM content_layout_serialized_file_dependencies d
 INNER JOIN content_layout_serialized_files_view src ON src.file_index = d.serialized_file_index
 INNER JOIN content_layout_serialized_files_view dep ON dep.file_index = d.dependency_index;
 
-CREATE VIEW IF NOT EXISTS content_layout_loadable_objects_view AS
--- Loadables resolved to their analyzed object. The object columns are NULL in a layout-only database.
-SELECT l.object_id_hash, l.guid, l.asset_path, l.lfid, l.is_root_asset,
-       f.content_hash || '.cf' AS filename,
-       o.id AS object, t.name AS type, o.name, o.size
-FROM content_layout_loadable_objects l
-LEFT JOIN content_layout_serialized_files f ON f.file_index = l.serialized_file_index
-LEFT JOIN objects o ON o.serialized_file = f.serialized_file AND o.object_id = l.output_lfid
-LEFT JOIN types t ON t.id = o.type;
-
 CREATE VIEW IF NOT EXISTS content_layout_binary_artifacts_view AS
 -- Artifacts with their on-disk filename derived from the content hash and category.
 SELECT artifact_index, content_hash, category, size,
@@ -48,11 +40,10 @@ FROM content_layout_binary_artifacts;
 
 CREATE VIEW IF NOT EXISTS content_layout_resource_files_view AS
 -- The data files (.resS/.resource) each content file uses, derived from the artifact graph.
-SELECT f.file_index, f.content_hash || '.cf' AS filename,
+SELECT f.file_index, f.filename,
        ra.category, rav.filename AS data_filename, ra.size
-FROM content_layout_serialized_files f
-INNER JOIN content_layout_binary_artifacts ca ON ca.content_hash = f.content_hash AND ca.category = 'contentfile'
-INNER JOIN content_layout_artifact_references r ON r.artifact_index = ca.artifact_index
+FROM content_layout_serialized_files_view f
+INNER JOIN content_layout_artifact_references r ON r.artifact_index = f.artifact_index
 INNER JOIN content_layout_binary_artifacts ra ON ra.artifact_index = r.referenced_artifact_index
 INNER JOIN content_layout_binary_artifacts_view rav ON rav.artifact_index = ra.artifact_index
 WHERE ra.category IN ('texture', 'mesh', 'audio', 'video');

Analyzer/Resources/Init.sql

@@ -168,7 +168,7 @@ WHERE m.type = 'Material';
 
 INSERT INTO types (id, name) VALUES (-1, 'Scene');
 
-PRAGMA user_version = 7;
+PRAGMA user_version = 8;
 
 PRAGMA synchronous = OFF;
 PRAGMA journal_mode = MEMORY;

Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableDependency.cs

@@ -12,7 +12,7 @@ internal class AddContentLayoutLoadableDependency : AbstractCommand
         protected override Dictionary<string, SqliteType> Fields => new()
         {
             { "serialized_file_index", SqliteType.Integer },
-            { "object_id_hash", SqliteType.Text }
+            { "loadable_index", SqliteType.Integer }
         };
     }
 }

Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableObject.cs

@@ -5,20 +5,40 @@ namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout
 {
     internal class AddContentLayoutLoadableObject : AbstractCommand
     {
+        // Version 2 layouts (Unity 6.6) record the source asset path and source lfid of each
+        // loadable; newer versions do not. The table is created with those columns only when a
+        // v2 layout is imported, so v3 databases are not cluttered with always-NULL columns.
+        // Set before CreateCommand, which reads DDLSource and Fields.
+        public bool V2Columns { get; set; }
+
         protected override string TableName => "content_layout_loadable_objects";
 
-        protected override string DDLSource => Properties.Resources.ContentLayoutLoadableObjects;
+        protected override string DDLSource => V2Columns
+            ? Properties.Resources.ContentLayoutLoadableObjectsV2
+            : Properties.Resources.ContentLayoutLoadableObjects;
 
-        protected override Dictionary<string, SqliteType> Fields => new()
+        protected override Dictionary<string, SqliteType> Fields
         {
-            { "object_id_hash", SqliteType.Text },
-            { "guid", SqliteType.Text },
-            { "asset_path", SqliteType.Text },
-            { "lfid", SqliteType.Integer },
-            { "identifier_type", SqliteType.Integer },
-            { "serialized_file_index", SqliteType.Integer },
-            { "output_lfid", SqliteType.Integer },
-            { "is_root_asset", SqliteType.Integer }
-        };
+            get
+            {
+                var fields = new Dictionary<string, SqliteType>
+                {
+                    { "loadable_index", SqliteType.Integer },
+                    { "guid", SqliteType.Text },
+                    { "lfid", SqliteType.Integer },
+                    { "identifier_type", SqliteType.Integer },
+                    { "serialized_file_index", SqliteType.Integer },
+                    { "is_root_asset", SqliteType.Integer }
+                };
+
+                if (V2Columns)
+                {
+                    fields.Add("asset_path", SqliteType.Text);
+                    fields.Add("source_lfid", SqliteType.Integer);
+                }
+
+                return fields;
+            }
+        }
     }
 }

Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSerializedFile.cs

@@ -12,9 +12,9 @@ internal class AddContentLayoutSerializedFile : AbstractCommand
         protected override Dictionary<string, SqliteType> Fields => new()
         {
             { "file_index", SqliteType.Integer },
-            { "cfid", SqliteType.Text },
+            { "stable_id", SqliteType.Text },
             { "is_builtin", SqliteType.Integer },
-            { "content_hash", SqliteType.Text },
+            { "artifact_index", SqliteType.Integer },
             { "serialized_file", SqliteType.Integer }
         };
     }

Analyzer/SQLite/Parsers/ContentLayoutParser.cs

@@ -11,7 +11,9 @@ namespace UnityDataTools.Analyzer.SQLite.Parsers
 {
     // Imports the ContentLayout.json produced by BuildPipeline.BuildContentDirectory into the
     // content_layout* tables (see Documentation/contentlayout.md for the file, and
-    // Documentation/contentlayout-database.md for the resulting schema).
+    // Documentation/contentlayout-database.md for the resulting schema). Version 2 files
+    // (Unity 6.6) are upgraded to the current in-memory model on the way in, so a single write
+    // path serves both supported versions.
     public class ContentLayoutParser : ISQLiteFileParser
     {
         private ContentLayoutSQLWriter m_Writer;
@@ -49,41 +51,73 @@ public bool CanParse(string filename)
 
         public void Parse(string filename)
         {
-            ContentLayout layout;
-            using (var reader = File.OpenText(filename))
-            {
-                var serializer = new JsonSerializer();
-                layout = (ContentLayout)serializer.Deserialize(reader, typeof(ContentLayout));
-            }
+            var version = BuildHistoryHelper.TryReadLayoutVersion(filename);
 
             // The tool's failure summary only includes exception details in verbose mode, so
             // report the reason for these expected failures directly.
-            if (layout == null)
+            if (version == null)
             {
-                Fail($"\"{filename}\" does not contain a ContentLayout.");
+                throw Fail($"\"{filename}\" does not contain a ContentLayout.");
             }
 
-            if (layout.Version != ContentLayout.CurrentVersion)
+            // The upgrader and the writer throw InvalidDataException for dangling references in
+            // the layout (an ObjectIdHash, ContentHash or ArtifactIndex with no target).
+            try
             {
-                Fail($"Unsupported ContentLayout.json version {layout.Version} (this version of UnityDataTool supports version {ContentLayout.CurrentVersion}).");
+                ContentLayout layout;
+                LoadableObjectV2Data[] v2Data = null;
+
+                switch (version)
+                {
+                    case Models.V2.ContentLayout.SchemaVersion:
+                        layout = ContentLayoutV2Upgrader.Upgrade(
+                            Deserialize<Models.V2.ContentLayout>(filename), out v2Data);
+                        break;
+
+                    case ContentLayout.CurrentVersion:
+                        layout = Deserialize<ContentLayout>(filename);
+                        break;
+
+                    default:
+                        throw Fail($"Unsupported ContentLayout.json version {version} (this version of UnityDataTool supports versions {Models.V2.ContentLayout.SchemaVersion} and {ContentLayout.CurrentVersion}).");
+                }
+
+                if (m_ImportedLayout != null)
+                {
+                    throw Fail($"Only a single ContentLayout.json can be analyzed (already imported \"{m_ImportedLayout}\").");
+                }
+
+                // Only create the content_layout tables when a layout is actually imported. The v2
+                // variant of the loadable-objects table carries extra source columns.
+                m_Writer.Init(v2Data != null);
+                m_Writer.WriteContentLayout(filename, layout, v2Data);
+                m_ImportedLayout = filename;
             }
+            catch (InvalidDataException e)
+            {
+                throw Fail($"\"{filename}\" is not a valid ContentLayout: {e.Message}");
+            }
+        }
 
-            if (m_ImportedLayout != null)
+        private T Deserialize<T>(string filename) where T : class
+        {
+            using var reader = File.OpenText(filename);
+            var serializer = new JsonSerializer();
+            var layout = (T)serializer.Deserialize(reader, typeof(T));
+
+            if (layout == null)
             {
-                Fail($"Only a single ContentLayout.json can be analyzed (already imported \"{m_ImportedLayout}\").");
+                throw Fail($"\"{filename}\" does not contain a ContentLayout.");
             }
 
-            // Only create the content_layout tables when a layout is actually imported.
-            m_Writer.Init();
-            m_Writer.WriteContentLayout(filename, layout);
-            m_ImportedLayout = filename;
+            return layout;
         }
 
-        private static void Fail(string message)
+        private static Exception Fail(string message)
         {
             Console.Error.WriteLine();
             Console.Error.WriteLine(message);
-            throw new Exception(message);
+            return new Exception(message);
         }
 
         // Called after all files are processed, so the analyzed .cf files all have their