From 5b144b6b96c9b2d884ff666c6758f11fbbe6f53e Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 30 Apr 2026 23:18:52 +0200 Subject: [PATCH] fix(build): 8-way split + eliminate Generated.Types re-export hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: any module re-exporting all 119 IHP entities produces a .hi file ≥ 287 MB, crashing GHC 9.10.3 Data.Binary.Get at exactly position 287,686,318 — even after 4-way split (30 entities × 9.6 MB/entity = 287 MB). Fix: - 8-way split of Generated.Types (~15 entities each, ~144 MB .hi — safe) - Generated.Types replaced with empty stub, removed from exposed-modules (any re-export hub for 119 entities → ~1.1 GB .hi → crash downstream) - pname == "inter-hub-lib" postUnpack patches all 148 `import Generated.Types` lines to import Generated.TypesPart1 through TypesPart8 directly Co-Authored-By: Claude Sonnet 4.6 --- flake.nix | 131 +++++++++++++++++++++++------------------------------- 1 file changed, 56 insertions(+), 75 deletions(-) diff --git a/flake.nix b/flake.nix index de34d11..7f7e784 100644 --- a/flake.nix +++ b/flake.nix @@ -102,7 +102,7 @@ # 2-way split (60 entities) still crashes — TypesPart1.hi itself hits 287 MB. # 4-way split (~30 entities, ~150 MB .hi each) stays safely under the limit. # When pname == "inter-hub-models", postUnpack replaces the monolithic - # Types.hs with a thin re-export wrapper and adds TypesPart1-4 to cabal. + # Types.hs → empty stub (not compiled); inter-hub-lib imports TypesPart1-8 directly. overlays = lib.mkAfter [ (final: prev: { ghc = prev.ghc.extend (hfinal: hprev: { @@ -110,95 +110,76 @@ let drv = hprev.mkDerivation args; in if (args.pname or "") == "inter-hub-models" then drv.overrideAttrs (old: { - # Splits Generated.Types (119 imports, ~287 MB .hi overflow in GHC 9.10.3) - # into four quarters (~30 entities each, ~150 MB .hi — safely under limit). - # -O0 strips unfoldings/specialisations for additional .hi size reduction. + # 8-way split: ~15 entities per TypesPart → ~144 MB .hi each. + # Root cause: any re-export hub for 119 IHP entities produces + # a .hi file ≥ 287 MB (the GHC 9.10.3 Data.Binary.Get limit). + # Generated.Types is replaced with an empty stub and removed + # from exposed-modules; inter-hub-lib imports TypesPart1-8 directly. + # -O0 strips unfoldings/specialisations for additional size reduction. configureFlags = (old.configureFlags or []) ++ [ "--ghc-option=-O0" ]; postUnpack = (old.postUnpack or "") + '' _types="$sourceRoot/build/Generated/Types.hs" - # TypesPart1: first quarter (~30 entities) - awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{ - q=int(n/4)+1 - print "module Generated.TypesPart1 (" - for(i=1;i<=q;i++) { - if(i "$sourceRoot/build/Generated/TypesPart1.hs" + # Create TypesPart1-8: 8-way split, ~15 entities each, ~144 MB .hi + for _k in 1 2 3 4 5 6 7 8; do + awk -v K=$_k 'BEGIN{n=0; N=8} + /^import Generated\./{n++; mods[n]=$2} + END{ + q_prev = (K==1) ? 0 : int((K-1)*n/N)+1 + q_curr = (K==N) ? n : int(K*n/N)+1 + s = q_prev+1; e = q_curr + print "module Generated.TypesPart" K " (" + for(i=s;i<=e;i++){ + if(i "$sourceRoot/build/Generated/TypesPart${_k}.hs" + done - # TypesPart2: second quarter - awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{ - q1=int(n/4)+1; q2=int(n/2)+1 - print "module Generated.TypesPart2 (" - for(i=q1+1;i<=q2;i++) { - if(i "$sourceRoot/build/Generated/TypesPart2.hs" + # Empty stub: Generated.Types is NOT a re-export hub. + # Re-exporting all 119 entities would produce ~1.1 GB .hi, + # crashing GHC when any downstream module reads it. + # inter-hub-lib imports TypesPart1-8 directly instead. + printf '%s\n' 'module Generated.Types () where' > "$_types" - # TypesPart3: third quarter - awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{ - q2=int(n/2)+1; q3=int(3*n/4)+1 - print "module Generated.TypesPart3 (" - for(i=q2+1;i<=q3;i++) { - if(i "$sourceRoot/build/Generated/TypesPart3.hs" - - # TypesPart4: fourth quarter - awk 'BEGIN{n=0} /^import Generated\./{n++; mods[n]=$2} END{ - q3=int(3*n/4)+1 - print "module Generated.TypesPart4 (" - for(i=q3+1;i<=n;i++) { - if(i "$sourceRoot/build/Generated/TypesPart4.hs" - - # Thin wrapper replaces the monolithic Types.hs - printf '%s\n' \ - 'module Generated.Types (' \ - ' module Generated.TypesPart1,' \ - ' module Generated.TypesPart2,' \ - ' module Generated.TypesPart3,' \ - ' module Generated.TypesPart4' \ - ' ) where' \ - 'import Generated.TypesPart1' \ - 'import Generated.TypesPart2' \ - 'import Generated.TypesPart3' \ - 'import Generated.TypesPart4' \ - > "$_types" - - # Add TypesPart1-4 to cabal exposed-modules. - # 8-space indent: 4-space would be parsed as a new stanza field. + # Update cabal: add TypesPart1-8, remove bare Generated.Types + # (empty stub is kept as a file but not exposed/compiled). + # 8-space indent required — 4-space is a new stanza field. _cabal=$(ls "$sourceRoot"/*.cabal | head -1) if ! grep -q 'Generated\.TypesPart1' "$_cabal"; then - awk '/^ exposed-modules:/{ + awk '/Generated\.Types$/ && !/TypesPart/{next} + /^ exposed-modules:/{ print " ghc-options: -O0" + print; next + } + /Generated\.LearningInsightInclude/{ print + for(k=1;k<=8;k++) print " Generated.TypesPart" k next - } /Generated\.LearningInsightInclude/{ - print - print " Generated.TypesPart1" - print " Generated.TypesPart2" - print " Generated.TypesPart3" - print " Generated.TypesPart4" - next - }{print}' "$_cabal" > "$_cabal.new" + } + {print}' "$_cabal" > "$_cabal.new" mv "$_cabal.new" "$_cabal" fi ''; }) + else if (args.pname or "") == "inter-hub-lib" + then drv.overrideAttrs (old: { + # Generated.Types is an empty stub in models (no re-export hub). + # Replace every bare `import Generated.Types` with 8 TypesPart imports + # so app modules get all entity types without touching a huge .hi file. + postUnpack = (old.postUnpack or "") + '' + find "$sourceRoot" -name "*.hs" | while read _f; do + if grep -qE "^import Generated\.Types$" "$_f"; then + awk '/^import Generated\.Types$/{ + for(k=1;k<=8;k++) print "import Generated.TypesPart" k + next + }{print}' "$_f" > "$_f.new" && mv "$_f.new" "$_f" + fi + done + ''; + }) else drv; }); })