Only re-dump package metadata when its dumpable content changed

migrations/2026_08_package_dump_requested_at.sql

@@ -0,0 +1,12 @@
+-- Run BEFORE deploying the code: purely additive, so the currently-deployed code keeps working while
+-- the new code needs both the column and the index to exist from the moment it starts. The matching
+-- drops live in 2026_08b_drop_dead_dump_columns.sql, which must run AFTER the deploy.
+--
+-- Marking a package for re-dump used to null dumpedAtV2, which is destructive: V2Dumper writes that
+-- once at the end of a run, so a mark landing mid-run was silently overwritten and never dumped.
+-- dumpRequestedAt records the request instead and staleness compares the two, so it cannot be lost.
+--
+-- Two statements on purpose: on its own the ADD COLUMN can run ALGORITHM=INSTANT, whereas folding it
+-- into the index DDL drags it down to an INPLACE rebuild of the whole package table.
+ALTER TABLE package ADD dumpRequestedAt DATETIME DEFAULT NULL;
+ALTER TABLE package ADD INDEX dumped2_requested_crawled_frozen_idx (dumpedAtV2, dumpRequestedAt, crawledAt, frozen);

migrations/2026_08b_drop_dead_dump_columns.sql

@@ -0,0 +1,10 @@
+-- Run AFTER the code is deployed: everything dropped here is still referenced by the previously
+-- deployed code. It reads and nulls package.dumpedAt, and its stale-package query names
+-- dumped2_crawled_frozen_idx in a USE INDEX hint, which errors out the moment the index is gone.
+-- The additive half runs before the deploy, see 2026_08_package_dump_requested_at.sql.
+--
+-- The v1 metadata dumper was removed, so package.dumpedAt is dead: nothing wrote it a non-null value
+-- and both of its readers had no callers. v2 staleness runs off dumpedAtV2 / dumpRequestedAt through
+-- dumped2_requested_crawled_frozen_idx, which supersedes dumped2_crawled_frozen_idx and (as dumpedAtV2
+-- leads it) the single-column dumped2_idx.
+ALTER TABLE package DROP INDEX dumped_idx, DROP COLUMN dumpedAt, DROP INDEX dumped2_crawled_frozen_idx, DROP INDEX dumped2_idx;

src/Controller/PackageController.php

@@ -1195,8 +1195,7 @@ public function abandonAction(Request $request, #[MapEntity] Package $package, #
             $package->setIndexedAt(null);
             $package->setCrawledAt(new \DateTimeImmutable());
             $package->setUpdatedAt(new \DateTimeImmutable());
-            $package->setDumpedAt(null);
-            $package->setDumpedAtV2(null);
+            $package->markForDump();
 
             $em = $this->getEM();
             $em->flush();
@@ -1222,8 +1221,7 @@ public function unabandonAction(#[MapEntity] Package $package, #[CurrentUser] ?U
         $package->setIndexedAt(null);
         $package->setCrawledAt(new \DateTimeImmutable());
         $package->setUpdatedAt(new \DateTimeImmutable());
-        $package->setDumpedAt(null);
-        $package->setDumpedAtV2(null);
+        $package->markForDump();
 
         $em = $this->getEM();
         $em->flush();

src/Entity/Job.php

@@ -140,6 +140,14 @@ public function getPayload(): array
         return $this->payload;
     }
 
+    /**
+     * @param T $payload
+     */
+    public function setPayload(array $payload): void
+    {
+        $this->payload = $payload;
+    }
+
     /**
      * @param self::STATUS_* $status
      */

src/Entity/Package.php

@@ -100,11 +100,9 @@ public static function casesForRole(bool $mayDisablePackages): array
 #[ORM\UniqueConstraint(name: 'package_name_idx', columns: ['name'])]
 #[ORM\Index(name: 'indexed_idx', columns: ['indexedAt'])]
 #[ORM\Index(name: 'crawled_idx', columns: ['crawledAt'])]
-#[ORM\Index(name: 'dumped_idx', columns: ['dumpedAt'])]
-#[ORM\Index(name: 'dumped2_idx', columns: ['dumpedAtV2'])]
 #[ORM\Index(name: 'repository_idx', columns: ['repository'])]
 #[ORM\Index(name: 'remoteid_idx', columns: ['remoteId'])]
-#[ORM\Index(name: 'dumped2_crawled_frozen_idx', columns: ['dumpedAtV2', 'crawledAt', 'frozen'])]
+#[ORM\Index(name: 'dumped2_requested_crawled_frozen_idx', columns: ['dumpedAtV2', 'dumpRequestedAt', 'crawledAt', 'frozen'])]
 #[ORM\Index(name: 'vendor_idx', columns: ['vendor'])]
 #[ORM\Index(name: 'frozen_idx', columns: ['frozen'])]
 #[ORM\Index(name: 'type_frozen_idx', columns: ['type', 'frozen'])]
@@ -190,10 +188,10 @@ class Package
     private ?\DateTimeImmutable $indexedAt = null;
 
     #[ORM\Column(type: 'datetime_immutable', nullable: true)]
-    private ?\DateTimeImmutable $dumpedAt = null;
+    private ?\DateTimeImmutable $dumpedAtV2 = null;
 
     #[ORM\Column(type: 'datetime_immutable', nullable: true)]
-    private ?\DateTimeImmutable $dumpedAtV2 = null;
+    private ?\DateTimeImmutable $dumpRequestedAt = null;
 
     /**
      * @var Collection<int, Download>&Selectable<int, Download>
@@ -693,24 +691,72 @@ public function getIndexedAt(): ?\DateTimeImmutable
         return $this->indexedAt;
     }
 
-    public function setDumpedAt(?\DateTimeImmutable $dumpedAt): void
+    /**
+     * For tests only — V2Dumper records dump times in bulk SQL, production code marks via
+     * markForDump() / forceDump(). Not nullable as a null dumpedAtV2 means "re-write no matter what".
+     */
+    public function setDumpedAtV2(\DateTimeImmutable $dumpedAt): void
     {
-        $this->dumpedAt = $dumpedAt;
+        $this->dumpedAtV2 = $dumpedAt;
     }
 
-    public function getDumpedAt(): ?\DateTimeImmutable
+    public function getDumpedAtV2(): ?\DateTimeImmutable
     {
-        return $this->dumpedAt;
+        return $this->dumpedAtV2;
     }
 
-    public function setDumpedAtV2(?\DateTimeImmutable $dumpedAt): void
+    /**
+     * Request a fresh metadata dump.
+     *
+     * Deliberately does not touch dumpedAtV2, which V2Dumper writes once at the end of a run: nulling
+     * it here would let an in-flight dump overwrite the mark and lose it. Recording the request
+     * separately keeps marking monotonic, so a mark landing mid-run survives.
+     */
+    public function markForDump(): void
     {
-        $this->dumpedAtV2 = $dumpedAt;
+        $this->dumpRequestedAt = new \DateTimeImmutable();
     }
 
-    public function getDumpedAtV2(): ?\DateTimeImmutable
+    /**
+     * Request a dump that re-writes the files even if their content is byte-identical, which busts the
+     * CDN cache and can shake a stuck storage replication loose. Use it for the deliberate "make it
+     * publish again" paths — a manual update, an unfreeze, a version being pulled or restored, and so
+     * on — not for ordinary content changes, which markForDump() covers at a fraction of the cost.
+     *
+     * The null dumpedAtV2 is what V2Dumper reads as "write this no matter what". Nulling is
+     * destructive, so this marks as well, or an in-flight dump run would overwrite the null and lose
+     * the request. That race can still cost the forcing itself, but never the re-dump.
+     */
+    public function forceDump(): void
     {
-        return $this->dumpedAtV2;
+        $this->dumpedAtV2 = null;
+        $this->markForDump();
+    }
+
+    public function getDumpRequestedAt(): ?\DateTimeImmutable
+    {
+        return $this->dumpRequestedAt;
+    }
+
+    /**
+     * Whether the next dump must re-write the files even if their content is unchanged. See forceDump().
+     */
+    public function isDumpForced(): bool
+    {
+        return $this->dumpedAtV2 === null;
+    }
+
+    /**
+     * Whether something requested a re-dump since the package was last dumped.
+     *
+     * Deliberately NOT a mirror of PackageRepository::getStalePackagesForDumpingV2(), which also sweeps
+     * in packages whose crawledAt is newer than their dump. Folding that clause in here would make
+     * every package "requested" and the metadata_dump.file{requested} tag useless.
+     */
+    public function isDumpRequested(): bool
+    {
+        return $this->dumpedAtV2 === null
+            || ($this->dumpRequestedAt !== null && $this->dumpRequestedAt >= $this->dumpedAtV2);
     }
 
     public function addMaintainer(User $maintainer): void
@@ -855,6 +901,10 @@ public function unfreeze(): void
         }
         $this->frozen = null;
         $this->setCrawledAt(null);
+        // Forcing regardless of the freeze reason: a suppressing freeze purged the published metadata,
+        // so it has to be written out again even though the content did not change. For the gentle
+        // reasons the re-write is redundant, but cheap insurance on a handful of packages.
+        $this->forceDump();
     }
 
     public function isFrozen(): bool

src/Entity/PackageRepository.php

@@ -77,23 +77,6 @@ public function findProviders(string $name): array
         return $result;
     }
 
-    /**
-     * @return array<string>
-     */
-    public function getPackageNamesUpdatedSince(\DateTimeInterface $date): array
-    {
-        $query = $this->getEntityManager()
-            ->createQuery('
-                SELECT p.name FROM App\Entity\Package p
-                WHERE p.dumpedAt >= :date AND (p.frozen IS NULL OR p.frozen NOT IN (:suppressed))
-            ')
-            ->setParameters(['date' => $date, 'suppressed' => PackageFreezeReason::suppressingCases()]);
-
-        $names = $this->getPackageNamesForQuery($query);
-
-        return array_map('strtolower', $names);
-    }
-
     /**
      * @return array<string>
      */
@@ -288,32 +271,21 @@ public function getStalePackagesForIndexing(): array
         return $conn->fetchAllAssociative('SELECT p.id FROM package p WHERE p.indexedAt IS NULL OR p.indexedAt <= p.crawledAt ORDER BY p.id ASC');
     }
 
-    /**
-     * @return list<int>
-     */
-    public function getStalePackagesForDumping(): array
-    {
-        $conn = $this->getEntityManager()->getConnection();
-
-        return $conn->fetchFirstColumn('
-            SELECT p.id
-            FROM package p
-            LEFT JOIN download d ON (d.id = p.id AND d.type = 1)
-            WHERE (p.dumpedAt IS NULL OR (p.dumpedAt <= p.crawledAt AND p.crawledAt < NOW()))
-            AND (p.frozen IS NULL OR p.frozen NOT IN (:suppressed))
-            AND (d.total > 1000 OR d.lastUpdated > :date)
-            ORDER BY p.crawledAt ASC
-        ', ['date' => date('Y-m-d H:i:s', strtotime('-4months')), 'suppressed' => PackageFreezeReason::suppressingValues()], ['suppressed' => ArrayParameterType::STRING]);
-    }
-
     /**
      * @return list<int>
      */
     public function getStalePackagesForDumpingV2(int $workerId = 0, int $numWorkers = 1): array
     {
         $conn = $this->getEntityManager()->getConnection();
 
-        $sql = 'SELECT p.id FROM package p USE INDEX (dumped2_crawled_frozen_idx) WHERE (p.dumpedAtV2 IS NULL OR (p.dumpedAtV2 <= p.crawledAt AND p.crawledAt < NOW())) AND (p.frozen IS NULL OR p.frozen NOT IN (:suppressed))';
+        // The >= is deliberate: both columns hold whole seconds, so a request landing in the same second
+        // as the dump is ambiguous and must count as stale — a redundant pass only costs a content
+        // comparison, a missed one is a lost change.
+        // The crawledAt clause is a transitional safety net, droppable once
+        // metadata_dump.file{result:written, requested:false} sits at ~0 outside --force runs.
+        // Both clauses compare column to column, so this scans the index rather than seeking it — hence
+        // the hint, and covering all four columns.
+        $sql = 'SELECT p.id FROM package p USE INDEX (dumped2_requested_crawled_frozen_idx) WHERE (p.dumpedAtV2 IS NULL OR p.dumpRequestedAt >= p.dumpedAtV2 OR (p.dumpedAtV2 <= p.crawledAt AND p.crawledAt < NOW())) AND (p.frozen IS NULL OR p.frozen NOT IN (:suppressed))';
         $params = ['suppressed' => PackageFreezeReason::suppressingValues()];
         $types = ['suppressed' => ArrayParameterType::STRING];
 

src/Entity/VersionRepository.php

@@ -69,6 +69,12 @@ public function remove(Version $version, bool $createAuditRecord = true, bool $a
         $package->getVersions()->removeElement($version);
         $package->setCrawledAt(new \DateTimeImmutable());
         $package->setUpdatedAt(new \DateTimeImmutable());
+        // A soft-deleted row was already excluded from the metadata, so hard-deleting it changes
+        // nothing. Worth skipping: the dominant caller is Updater's prune loop purging dev rows a day
+        // after soft-deleting them, i.e. routine branch churn.
+        if (!$version->isSoftDeleted()) {
+            $package->markForDump();
+        }
         $em->persist($package);
 
         $this->versionIdCache->deleteVersion($package, $version);
@@ -108,10 +114,19 @@ public function softDelete(Version $version, VersionDeletionReason $reason, ?str
 
         $em->persist(AuditRecord::versionSoftDeleted($version, $reason, $reasonText, $internalReasonText, $actor));
 
+        // Mark directly rather than leaning on the scheduled job: pulling a version is the security
+        // path, and the job's force_dump only lands if the crawl succeeds, so a repository that 404s
+        // would leave the pulled version published indefinitely. The dumper excludes soft-deleted
+        // versions by itself, so there is nothing to wait for.
+        $version->getPackage()->markForDump();
+
         if (!$version->getPackage()->isFrozen()) {
-            $this->scheduler->scheduleUpdate($version->getPackage(), 'version_recover');
+            // still schedule the crawl so dependents/suggesters get recomputed without this version
+            $this->scheduler->scheduleUpdate($version->getPackage(), 'version_soft_delete', forceDump: true);
         } else {
-            $version->getPackage()->setCrawledAt(new \DateTimeImmutable());
+            // The mark above replaces the crawledAt bump this used to do, which also means no search
+            // reindex (getStalePackagesForIndexing() keys off indexedAt <= crawledAt). Fine, as the
+            // search document is package-level and unaffected by pulling one version.
             $this->getEntityManager()->persist($version->getPackage());
         }
     }
@@ -132,7 +147,10 @@ public function recover(Version $version, ?User $actor): void
 
         $em->persist(AuditRecord::versionRecovered($version, $previousReason, $actor));
 
-        $this->scheduler->scheduleUpdate($version->getPackage(), 'version_recover');
+        // marked directly for the same reason as softDelete(): the re-dump must not depend on a
+        // successful crawl, or on the job surviving Scheduler dedup
+        $version->getPackage()->markForDump();
+        $this->scheduler->scheduleUpdate($version->getPackage(), 'version_recover', forceDump: true);
     }
 
     /**

src/EventListener/SecurityAdvisoryUpdateListener.php

@@ -63,10 +63,16 @@ public function postRemove(SecurityAdvisory $advisory, LifecycleEventArgs $event
 
     public function flushChangesToPackages(): void
     {
+        if (\count($this->packagesToMarkStale) === 0) {
+            return;
+        }
+
         $packageNames = array_keys($this->packagesToMarkStale);
-        $pkg = $this->getEM()->getConnection()->executeStatement(
-            'UPDATE package SET dumpedAtV2 = null WHERE name IN (:names)',
-            ['names' => $packageNames],
+        // Records the request rather than nulling dumpedAtV2, so an in-flight dumper run cannot
+        // overwrite the mark. App clock, to match the one V2Dumper compares against.
+        $this->getEM()->getConnection()->executeStatement(
+            'UPDATE package SET dumpRequestedAt = :now WHERE name IN (:names)',
+            ['now' => date('Y-m-d H:i:s'), 'names' => $packageNames],
             ['names' => ArrayParameterType::STRING]
         );
 

src/FilterList/FilterListEntryUpdateListener.php

@@ -66,9 +66,11 @@ public function flushChangesToPackages(): void
         }
 
         $packageNames = array_keys($this->packagesToMarkStale);
+        // Records the request rather than nulling dumpedAtV2, so an in-flight dumper run cannot
+        // overwrite the mark. App clock, to match the one V2Dumper compares against.
         $this->getEM()->getConnection()->executeStatement(
-            'UPDATE package SET dumpedAtV2 = null WHERE name IN (:names)',
-            ['names' => $packageNames],
+            'UPDATE package SET dumpRequestedAt = :now WHERE name IN (:names)',
+            ['now' => date('Y-m-d H:i:s'), 'names' => $packageNames],
             ['names' => ArrayParameterType::STRING]
         );
 

src/Form/Type/AbandonedType.php

@@ -15,6 +15,7 @@
 use Symfony\Component\Form\AbstractType;
 use Symfony\Component\Form\Extension\Core\Type\TextType;
 use Symfony\Component\Form\FormBuilderInterface;
+use Symfony\Component\Validator\Constraints\Length;
 
 /**
  * Class AbandonedType
@@ -34,6 +35,9 @@ public function buildForm(FormBuilderInterface $builder, array $options): void
                 'required' => false,
                 'label' => 'Replacement package',
                 'attr' => ['placeholder' => 'optional package name'],
+                // Package::$replacementPackage is a 255 char column, so anything longer would fail the
+                // flush with a 500 instead of showing the user what is wrong
+                'constraints' => [new Length(max: 255)],
             ]
         );
     }