Skip to content

[#99] Resolve ContentDirectory references when content files lack the .cf extension #105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
SkowronskiAndrew wants to merge 1 commit into
base: main
Choose a base branch
from
+47 −25
Draft

[#99] Resolve ContentDirectory references when content files lack the .cf extension #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 19 additions & 14 deletions Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,11 @@ public void WriteContentLayout(string filename, ContentLayout layout)
}

// Fills the shared dependency map that resolves the external references of the analyzed
// .cf files (see ContentFileDependencyMap): for each content file, the dependency indexes
// become the target filenames, in order. Built-in dependencies (no content hash) map to
// null so the reference falls back to the external table path.
// content files (see ContentFileDependencyMap): for each content file, the dependency
// indexes become the target names, in order. Built-in dependencies (no content hash) map to
// null so the reference falls back to the external table path. Keys and resolved names use
// the normalized content-hash name so they match the analyzed files regardless of whether
// those carry the ".cf" extension on disk.
private void PopulateContentFileDependencies(ContentLayout layout)
{
var hashByIndex = (layout.SerializedFiles ?? []).ToDictionary(f => f.Index, f => f.ContentHash);
Expand All @@ -203,21 +205,24 @@ private void PopulateContentFileDependencies(ContentLayout layout)

var resolved = (file.SerializedFileDependencies ?? [])
.Select(i => hashByIndex.TryGetValue(i, out var hash) && !string.IsNullOrEmpty(hash)
? hash + ".cf"
? ContentFileDependencyMap.NormalizeFileName(hash)
: null)
.ToArray();

m_ContentFileDependencies.Add(file.ContentHash + ".cf", resolved);
m_ContentFileDependencies.Add(ContentFileDependencyMap.NormalizeFileName(file.ContentHash), resolved);
}
}

// Fills in the serialized_file column, linking each layout entry to its serialized_files
// row. Called after all files are processed. Ids come from the shared IdProvider, so
// entries whose .cf file was analyzed link to the row the analyze pass wrote (loose or
// inside an archive, with its archive column intact). For files never encountered on the
// input (a layout-only analyze, or a subset of a build) a placeholder serialized_files
// row is written - name only, archive NULL, no objects - so the link is always valid,
// mirroring what the dangling-refs finalize does for referenced-but-unanalyzed files.
// row. Called after all files are processed. Ids come from the shared IdProvider (keyed by
// the normalized content hash), so entries whose content file was analyzed link to the row
// the analyze pass wrote (loose or inside an archive, with its archive column intact),
// regardless of whether that file carried the ".cf" extension on disk. For files never
// encountered on the input (a layout-only analyze, or a subset of a build) a placeholder
// serialized_files row is written - name only, archive NULL, no objects - so the link is
// always valid, mirroring what the dangling-refs finalize does for referenced-but-unanalyzed
// files. The placeholder's name uses the canonical "<hash>.cf" form since the file was not
// seen on disk.
public void LinkSerializedFiles()
{
var existingIds = new HashSet<int>();
Expand Down Expand Up @@ -246,8 +251,8 @@ public void LinkSerializedFiles()
{
foreach (var file in m_ImportedFiles)
{
var fileName = file.ContentHash + ".cf";
var id = m_SerializedFileIdProvider.GetId(fileName);
var id = m_SerializedFileIdProvider.GetId(
ContentFileDependencyMap.NormalizeFileName(file.ContentHash));

update.Parameters["@id"].Value = id;
update.Parameters["@file_index"].Value = file.Index;
Expand All @@ -257,7 +262,7 @@ public void LinkSerializedFiles()
{
addStubRow.SetValue("id", id);
addStubRow.SetValue("archive", null);
addStubRow.SetValue("name", fileName);
addStubRow.SetValue("name", file.ContentHash + ".cf");
addStubRow.ExecuteNonQuery();
}
}
Expand Down
15 changes: 10 additions & 5 deletions Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ public class SerializedFileSQLiteWriter : IDisposable
private bool m_SkipCrc;

// Global id assignment shared across every serialized file in the database.
// m_SerializedFileIdProvider maps a serialized file (by lowercased file name) to its
// serialized_files row id; it is owned by AnalyzerTool because the ContentLayout import
// m_SerializedFileIdProvider maps a serialized file (by file name, matched case-sensitively as
// it appears on disk / in the archive, with any ".cf" content-file extension normalized away)
// to its serialized_files row id; it is owned by AnalyzerTool because the ContentLayout import
// assigns ids through the same provider. m_ObjectIdProvider maps a (serialized file id,
// pathId) pair to its objects row id. See ObjectIdProvider for how cross-file references
// are resolved.
Expand Down Expand Up @@ -185,7 +186,8 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
using var sf = UnityFileSystem.OpenSerializedFile(fullPath);
using var reader = new UnityFileReader(fullPath, 64 * 1024 * 1024);
using var pptrReader = new PPtrAndCrcProcessor(sf, reader, containingFolder, m_SkipCrc, AddReference);
int serializedFileId = m_SerializedFileIdProvider.GetId(Path.GetFileName(fullPath));
int serializedFileId = m_SerializedFileIdProvider.GetId(
ContentFileDependencyMap.NormalizeFileName(Path.GetFileName(fullPath)));
int sceneId = -1;

// Two SerializedFiles with the same name map to the same id (the provider deduplicates by
Expand Down Expand Up @@ -278,7 +280,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
// dependency list, matched by position; built-in dependencies (null entries) keep the
// external table path.
var resolvedDependencies = m_ContentFileDependencies.GetDependencies(
Path.GetFileName(fullPath));
ContentFileDependencyMap.NormalizeFileName(Path.GetFileName(fullPath)));

if (resolvedDependencies != null && resolvedDependencies.Length != sf.ExternalReferences.Count)
{
Expand All @@ -292,8 +294,11 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
m_LocalToDbFileId.Add(localId++, serializedFileId);
foreach (var extRef in sf.ExternalReferences)
{
// Resolved dependency names are already normalized content hashes; normalize the
// external-table fallback too so a target keys the same way it does when analyzed.
var name = resolvedDependencies?[localId - 1]
?? extRef.Path.Substring(extRef.Path.LastIndexOf('/') + 1);
?? ContentFileDependencyMap.NormalizeFileName(
extRef.Path.Substring(extRef.Path.LastIndexOf('/') + 1));
m_LocalToDbFileId.Add(localId++, m_SerializedFileIdProvider.GetId(name));
}

Expand Down
24 changes: 18 additions & 6 deletions Analyzer/Util/ContentFileDependencyMap.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;

namespace UnityDataTools.Analyzer.Util;
Expand All @@ -6,10 +7,10 @@ namespace UnityDataTools.Analyzer.Util;
// In ContentDirectory builds the external reference table inside a SerializedFile holds symbolic
// .cfid placeholders instead of real filenames; the actual target of a reference is determined
// positionally through the layout's dependency list (see Documentation/contentdirectory-format.md).
// This map holds, for each content file (keyed by its "<contenthash>.cf" filename), the resolved
// dependency filenames in external-reference-table order. An entry is null where the
// dependency is a built-in file (which has no content hash); such references fall back to the
// path from the external reference table.
// This map holds, for each content file (keyed by its normalized content-hash name, see
// NormalizeFileName), the resolved dependency names in external-reference-table order. An entry is
// null where the dependency is a built-in file (which has no content hash); such references fall
// back to the path from the external reference table.
public class ContentFileDependencyMap
{
private Dictionary<string, string[]> m_Dependencies = new();
Expand All @@ -19,10 +20,21 @@ public void Add(string fileName, string[] resolvedDependencies)
m_Dependencies[fileName] = resolvedDependencies;
}

// Returns the resolved dependency filenames for the given filename, or null if the file is
// not covered by an imported ContentLayout.
// Returns the resolved dependency names for the given (normalized) content-file name, or null
// if the file is not covered by an imported ContentLayout.
public string[] GetDependencies(string fileName)
{
return m_Dependencies.TryGetValue(fileName, out var dependencies) ? dependencies : null;
}

// Content files are named by their content hash and may or may not carry the ".cf" extension:
// loose builds and archived builds vary, and the extension is informational (see
// Documentation/contentdirectory-format.md). Strip it so a content file resolves to the same
// serialized-file id whether or not the extension is present on disk. Non-content files never
// end in ".cf", so this is a no-op for them. The name's case is preserved, matching how
// serialized-file names are keyed case-sensitively elsewhere.
public static string NormalizeFileName(string fileName)
{
return fileName.EndsWith(".cf", StringComparison.OrdinalIgnoreCase) ? fileName[..^3] : fileName;
}
}
Loading