generated from coulomb/repo-seed
fix(build): 8-way split + eliminate Generated.Types re-export hub
Some checks failed
Build and Deploy / build-push-deploy (push) Has been cancelled
Some checks failed
Build and Deploy / build-push-deploy (push) Has been cancelled
Root cause: any module re-exporting all 119 IHP entities produces a .hi file ≥ 287 MB, crashing GHC 9.10.3 Data.Binary.Get at exactly position 287,686,318 — even after 4-way split (30 entities × 9.6 MB/entity = 287 MB). Fix: - 8-way split of Generated.Types (~15 entities each, ~144 MB .hi — safe) - Generated.Types replaced with empty stub, removed from exposed-modules (any re-export hub for 119 entities → ~1.1 GB .hi → crash downstream) - pname == "inter-hub-lib" postUnpack patches all 148 `import Generated.Types` lines to import Generated.TypesPart1 through TypesPart8 directly Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
131
flake.nix
131
flake.nix
@@ -102,7 +102,7 @@
|
||||
# 2-way split (60 entities) still crashes — TypesPart1.hi itself hits 287 MB.
|
||||
# 4-way split (~30 entities, ~150 MB .hi each) stays safely under the limit.
|
||||
# When pname == "inter-hub-models", postUnpack replaces the monolithic
|
||||
# Types.hs with a thin re-export wrapper and adds TypesPart1-4 to cabal.
|
||||
# Types.hs → empty stub (not compiled); inter-hub-lib imports TypesPart1-8 directly.
|
||||
overlays = lib.mkAfter [
|
||||
(final: prev: {
|
||||
ghc = prev.ghc.extend (hfinal: hprev: {
|
||||
@@ -110,95 +110,76 @@
|
||||
let drv = hprev.mkDerivation args;
|
||||
in if (args.pname or "") == "inter-hub-models"
|
||||
then drv.overrideAttrs (old: {
|
||||
# Splits Generated.Types (119 imports, ~287 MB .hi overflow in GHC 9.10.3)
|
||||
# into four quarters (~30 entities each, ~150 MB .hi — safely under limit).
|
||||
# -O0 strips unfoldings/specialisations for additional .hi size reduction.
|
||||
# 8-way split: ~15 entities per TypesPart → ~144 MB .hi each.
|
||||
# Root cause: any re-export hub for 119 IHP entities produces
|
||||
# a .hi file ≥ 287 MB (the GHC 9.10.3 Data.Binary.Get limit).
|
||||
# Generated.Types is replaced with an empty stub and removed
|
||||
# from exposed-modules; inter-hub-lib imports TypesPart1-8 directly.
|
||||
# -O0 strips unfoldings/specialisations for additional size reduction.
|
||||
configureFlags = (old.configureFlags or []) ++ [ "--ghc-option=-O0" ];
|
||||
postUnpack = (old.postUnpack or "") + ''
|
||||
_types="$sourceRoot/build/Generated/Types.hs"
|
||||
|
||||
# TypesPart1: first quarter (~30 entities)
|
||||
awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{
|
||||
q=int(n/4)+1
|
||||
print "module Generated.TypesPart1 ("
|
||||
for(i=1;i<=q;i++) {
|
||||
if(i<q) print " module " mods[i] ","
|
||||
else print " module " mods[i]
|
||||
}
|
||||
print " ) where"
|
||||
for(i=1;i<=q;i++) print "import " mods[i]
|
||||
}' "$_types" > "$sourceRoot/build/Generated/TypesPart1.hs"
|
||||
# Create TypesPart1-8: 8-way split, ~15 entities each, ~144 MB .hi
|
||||
for _k in 1 2 3 4 5 6 7 8; do
|
||||
awk -v K=$_k 'BEGIN{n=0; N=8}
|
||||
/^import Generated\./{n++; mods[n]=$2}
|
||||
END{
|
||||
q_prev = (K==1) ? 0 : int((K-1)*n/N)+1
|
||||
q_curr = (K==N) ? n : int(K*n/N)+1
|
||||
s = q_prev+1; e = q_curr
|
||||
print "module Generated.TypesPart" K " ("
|
||||
for(i=s;i<=e;i++){
|
||||
if(i<e) print " module " mods[i] ","
|
||||
else print " module " mods[i]
|
||||
}
|
||||
print " ) where"
|
||||
for(i=s;i<=e;i++) print "import " mods[i]
|
||||
}' "$_types" > "$sourceRoot/build/Generated/TypesPart${_k}.hs"
|
||||
done
|
||||
|
||||
# TypesPart2: second quarter
|
||||
awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{
|
||||
q1=int(n/4)+1; q2=int(n/2)+1
|
||||
print "module Generated.TypesPart2 ("
|
||||
for(i=q1+1;i<=q2;i++) {
|
||||
if(i<q2) print " module " mods[i] ","
|
||||
else print " module " mods[i]
|
||||
}
|
||||
print " ) where"
|
||||
for(i=q1+1;i<=q2;i++) print "import " mods[i]
|
||||
}' "$_types" > "$sourceRoot/build/Generated/TypesPart2.hs"
|
||||
# Empty stub: Generated.Types is NOT a re-export hub.
|
||||
# Re-exporting all 119 entities would produce ~1.1 GB .hi,
|
||||
# crashing GHC when any downstream module reads it.
|
||||
# inter-hub-lib imports TypesPart1-8 directly instead.
|
||||
printf '%s\n' 'module Generated.Types () where' > "$_types"
|
||||
|
||||
# TypesPart3: third quarter
|
||||
awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{
|
||||
q2=int(n/2)+1; q3=int(3*n/4)+1
|
||||
print "module Generated.TypesPart3 ("
|
||||
for(i=q2+1;i<=q3;i++) {
|
||||
if(i<q3) print " module " mods[i] ","
|
||||
else print " module " mods[i]
|
||||
}
|
||||
print " ) where"
|
||||
for(i=q2+1;i<=q3;i++) print "import " mods[i]
|
||||
}' "$_types" > "$sourceRoot/build/Generated/TypesPart3.hs"
|
||||
|
||||
# TypesPart4: fourth quarter
|
||||
awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{
|
||||
q3=int(3*n/4)+1
|
||||
print "module Generated.TypesPart4 ("
|
||||
for(i=q3+1;i<=n;i++) {
|
||||
if(i<n) print " module " mods[i] ","
|
||||
else print " module " mods[i]
|
||||
}
|
||||
print " ) where"
|
||||
for(i=q3+1;i<=n;i++) print "import " mods[i]
|
||||
}' "$_types" > "$sourceRoot/build/Generated/TypesPart4.hs"
|
||||
|
||||
# Thin wrapper replaces the monolithic Types.hs
|
||||
printf '%s\n' \
|
||||
'module Generated.Types (' \
|
||||
' module Generated.TypesPart1,' \
|
||||
' module Generated.TypesPart2,' \
|
||||
' module Generated.TypesPart3,' \
|
||||
' module Generated.TypesPart4' \
|
||||
' ) where' \
|
||||
'import Generated.TypesPart1' \
|
||||
'import Generated.TypesPart2' \
|
||||
'import Generated.TypesPart3' \
|
||||
'import Generated.TypesPart4' \
|
||||
> "$_types"
|
||||
|
||||
# Add TypesPart1-4 to cabal exposed-modules.
|
||||
# 8-space indent: 4-space would be parsed as a new stanza field.
|
||||
# Update cabal: add TypesPart1-8, remove bare Generated.Types
|
||||
# (empty stub is kept as a file but not exposed/compiled).
|
||||
# 8-space indent required — 4-space is a new stanza field.
|
||||
_cabal=$(ls "$sourceRoot"/*.cabal | head -1)
|
||||
if ! grep -q 'Generated\.TypesPart1' "$_cabal"; then
|
||||
awk '/^ exposed-modules:/{
|
||||
awk '/Generated\.Types$/ && !/TypesPart/{next}
|
||||
/^ exposed-modules:/{
|
||||
print " ghc-options: -O0"
|
||||
print; next
|
||||
}
|
||||
/Generated\.LearningInsightInclude/{
|
||||
print
|
||||
for(k=1;k<=8;k++) print " Generated.TypesPart" k
|
||||
next
|
||||
} /Generated\.LearningInsightInclude/{
|
||||
print
|
||||
print " Generated.TypesPart1"
|
||||
print " Generated.TypesPart2"
|
||||
print " Generated.TypesPart3"
|
||||
print " Generated.TypesPart4"
|
||||
next
|
||||
}{print}' "$_cabal" > "$_cabal.new"
|
||||
}
|
||||
{print}' "$_cabal" > "$_cabal.new"
|
||||
mv "$_cabal.new" "$_cabal"
|
||||
fi
|
||||
'';
|
||||
})
|
||||
else if (args.pname or "") == "inter-hub-lib"
|
||||
then drv.overrideAttrs (old: {
|
||||
# Generated.Types is an empty stub in models (no re-export hub).
|
||||
# Replace every bare `import Generated.Types` with 8 TypesPart imports
|
||||
# so app modules get all entity types without touching a huge .hi file.
|
||||
postUnpack = (old.postUnpack or "") + ''
|
||||
find "$sourceRoot" -name "*.hs" | while read _f; do
|
||||
if grep -qE "^import Generated\.Types$" "$_f"; then
|
||||
awk '/^import Generated\.Types$/{
|
||||
for(k=1;k<=8;k++) print "import Generated.TypesPart" k
|
||||
next
|
||||
}{print}' "$_f" > "$_f.new" && mv "$_f.new" "$_f"
|
||||
fi
|
||||
done
|
||||
'';
|
||||
})
|
||||
else drv;
|
||||
});
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user