package deploy import ( "context" "errors" "io/fs" "os" "path/filepath" "strconv" "strings" "time" "github.com/iceBear67/simplepages/internal/cas" "github.com/iceBear67/simplepages/internal/store" ) // staleUploadAge is how long a deployment may sit unfinished before recovery // gives up on it. Generous on purpose: the only thing separating a CI job that // died from one that is uploading a large site over a slow link is how long it // has been, and expiring the second kind turns a slow deploy into a failed one. const staleUploadAge = 24 * time.Hour // Recover makes the filesystem and the database agree again after a crash or an // unclean shutdown. // // It runs once at startup, before any listener exists and before the registry is // built, so it can assume it is the only thing touching either. Everything it // does is idempotent: being killed halfway through only means the next start // finds a little more to do. // // Nothing here is allowed to be fatal. A server that refuses to start because // one directory could not be swept is worse than one that starts and logs it — // the deployments themselves are already durable, and every inconsistency this // looks for is one the running system tolerates. func (s *Service) Recover(ctx context.Context) error { // An upload that was in progress is unreferenced by construction: a blob only // becomes reachable by being renamed out of the temp directory. if n, err := s.CAS.PurgeTemp(); err != nil { s.Log.Warn("could not clear interrupted uploads", "err", err) } else if n > 0 { s.Log.Info("cleared interrupted uploads", "count", n) } if err := s.recoverBlobs(ctx); err != nil { return err } // Then the deployments that were still being written when the process went // away, so their manifest rows are gone before the blob collector next runs // and can reclaim whatever only they referenced. n, err := s.DB.ExpireStaleDeployments(ctx, time.Now().Add(-staleUploadAge), "abandoned: no activity for "+staleUploadAge.String()) if err != nil { return err } if n > 0 { s.Log.Info("expired unfinished deployments", "count", n) } if err := s.resumeDeletions(ctx); err != nil { return err } return s.sweepTrees(ctx) } // recoverBlobs finds content the database believes is on disk and is not. // // This is the "restored the database, lost the disk" case, and also what a // half-finished collector sweep leaves behind. Marking the rows absent is // enough to fix it: the next deploy naming one of these digests is asked to // upload it, and every deployment that referenced it becomes deployable again as // soon as one does. func (s *Service) recoverBlobs(ctx context.Context) error { var absent []cas.Digest err := s.DB.EachPresentBlob(ctx, func(d cas.Digest) error { has, err := s.CAS.Has(d) if err != nil { return err } if !has { absent = append(absent, d) } return nil }) if err != nil { return err } if len(absent) == 0 { return nil } if err := s.DB.MarkBlobsAbsent(ctx, absent); err != nil { return err } // Loud, because on a healthy server this number is zero. Anything else means // the content store lost data, and an operator wants to hear about it before // a deploy fails for a reason that looks like the client's fault. s.Log.Warn("content is missing from the store and was marked for re-upload", "blobs", len(absent)) return nil } // resumeDeletions finishes what a collector sweep was doing when it stopped. // // A deployment enters the deleting state before anything of it is removed, so a // row still in that state is a tree that may be half gone. Half a tree is // exactly what nothing may serve, which is why the state is committed first: it // makes an interrupted deletion recognisable rather than indistinguishable from // a healthy deployment. func (s *Service) resumeDeletions(ctx context.Context) error { deps, err := s.DB.DeploymentsInState(ctx, store.StateDeleting, 0) if err != nil { return err } for _, dep := range deps { if err := s.removeDeployment(ctx, dep); err != nil { s.Log.Warn("could not finish deleting a deployment", "deployment", dep.PublicID, "err", err) continue } s.Log.Info("finished deleting a deployment", "deployment", dep.PublicID) } return nil } // removeDeployment takes a claimed deployment the rest of the way: its tree // first, then its rows. Tree before rows, because a row without a tree is a // deployment that simply cannot be activated, whereas a tree without a row is // disk nobody will ever account for again. func (s *Service) removeDeployment(ctx context.Context, dep *store.Deployment) error { if s.Dir != "" { dir := DeploymentDir(s.Dir, dep.ProjectID, dep.PublicID) if err := os.RemoveAll(dir); err != nil { return err } if err := os.RemoveAll(dir + stagingSuffix); err != nil { return err } } return s.DB.DeleteDeployment(ctx, dep.ID) } // sweepTrees removes assembled trees that nothing refers to. // // Two kinds: staging directories, which are by definition a build that never // finished, and directories whose deployment row is gone — the reverse of the // blob audit above, and the residue of a deletion that removed rows before it // removed files, or of a database restored from an older backup than the disk. func (s *Service) sweepTrees(ctx context.Context) error { if s.Dir == "" { return nil } refs, err := s.DB.AllDeploymentRefs(ctx) if err != nil { return err } known := make(map[store.DeploymentRef]struct{}, len(refs)) for _, r := range refs { known[r] = struct{}{} } projects, err := os.ReadDir(s.Dir) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return err } var staging, orphans int for _, pe := range projects { // Only the layout this package writes is ever considered for removal: // one directory per project id, named by the id. Anything else under // $DATA_DIR/deployments was put there by someone else and is left alone. projectID, err := strconv.ParseInt(pe.Name(), 10, 64) if err != nil || !pe.IsDir() { continue } projectDir := filepath.Join(s.Dir, pe.Name()) entries, err := os.ReadDir(projectDir) if err != nil { s.Log.Warn("could not read a project's deployment directory", "dir", projectDir, "err", err) continue } for _, e := range entries { if !e.IsDir() { continue } name := e.Name() target := filepath.Join(projectDir, name) switch { case strings.HasSuffix(name, stagingSuffix): if err := os.RemoveAll(target); err != nil { s.Log.Warn("could not remove a staging directory", "dir", target, "err", err) continue } staging++ default: if _, ok := known[store.DeploymentRef{ProjectID: projectID, PublicID: name}]; ok { continue } if err := os.RemoveAll(target); err != nil { s.Log.Warn("could not remove an orphaned deployment tree", "dir", target, "err", err) continue } orphans++ } } } if staging > 0 || orphans > 0 { s.Log.Info("swept unreferenced deployment trees", "staging", staging, "orphaned", orphans) } return nil }