[feature] add statuses multi GET endpoint (#4814)

this follows on from the work of @cdn0x12 in https://codeberg.org/superseriousbusiness/gotosocial/pulls/4795 but modifies the implementation to better make use of our multi lookup database and cache calls.

in addition to the above, this moves much of the surfacing (timelining and notifying) logic to derefencer hooks that are now setup to be called on dereference of accounts, statuses, media and emojis. though only statuses and media are hooked up for now.

Co-authored-by: cdn0x12 <git@cdn0x12.dev>
Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4814

TODO.md

@@ -13,6 +13,7 @@ things noted down by the maintainers that could do with being done!
 - [performance] add multi-delete queries for mentions, statuses (+boosts), etc
 
 ## miscellaneous
+- [chore] move dereferencer hooks to Ingester{} type that handles calling appropriate hooks for any ingested model
 - [chore] finish code commenting where missing (search for '// (\w+\b)?...')
 - [chore] move away from using Gin, they're all-in on "AI", blegh
 - [chore] deinterface the database somehow (where possible given dependency cycling 😭), have a single DB type so all bundb/*.go can access all other internal funcs

docs/api/swagger.yaml

@@ -14755,6 +14755,56 @@ paths:
             tags:
                 - scheduled_statuses
     /api/v1/statuses:
+        get:
+            operationId: statusesGet
+            parameters:
+                - collectionFormat: multi
+                  description: Target status IDs.
+                  in: query
+                  items:
+                    type: string
+                  name: id[]
+                  required: true
+                  type: array
+            produces:
+                - application/json
+            responses:
+                "200":
+                    description: The requested statuses.
+                    schema:
+                        items:
+                            $ref: '#/definitions/status'
+                        type: array
+                "400":
+                    description: bad request
+                    schema:
+                        $ref: '#/definitions/error'
+                "401":
+                    description: unauthorized
+                    schema:
+                        $ref: '#/definitions/error'
+                "403":
+                    description: forbidden
+                    schema:
+                        $ref: '#/definitions/error'
+                "406":
+                    description: not acceptable
+                    schema:
+                        $ref: '#/definitions/error'
+                "422":
+                    description: unprocessable entity
+                    schema:
+                        $ref: '#/definitions/error'
+                "500":
+                    description: internal server error
+                    schema:
+                        $ref: '#/definitions/error'
+            security:
+                - OAuth2 Bearer:
+                    - read:statuses
+            summary: View multiple statuses with the given IDs.
+            tags:
+                - statuses
         post:
             consumes:
                 - application/json

internal/api/client/statuses/status.go

@@ -86,6 +86,7 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
 	// create / get / edit / delete status
 	attachHandler(http.MethodPost, BasePath, m.StatusCreatePOSTHandler)
 	attachHandler(http.MethodGet, BasePathWithID, m.StatusGETHandler)
+	attachHandler(http.MethodGet, BasePath, m.StatusesGETHandler)
 	attachHandler(http.MethodPut, BasePathWithID, m.StatusEditPUTHandler)
 	attachHandler(http.MethodDelete, BasePathWithID, m.StatusDELETEHandler)
 

internal/api/client/statuses/statusesget.go

@@ -0,0 +1,125 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package statuses
+
+import (
+	"errors"
+	"net/http"
+
+	apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
+	"code.superseriousbusiness.org/gotosocial/internal/gtserror"
+	"github.com/gin-gonic/gin"
+)
+
+// StatusesGETHandler swagger:operation GET /api/v1/statuses statusesGet
+//
+// View multiple statuses with the given IDs.
+//
+//	---
+//	tags:
+//	- statuses
+//
+//	produces:
+//	- application/json
+//
+//	parameters:
+//	-
+//		name: id[]
+//		type: array
+//		items:
+//			type: string
+//		description: Target status IDs.
+//		in: query
+//		collectionFormat: multi
+//		required: true
+//
+//	security:
+//	- OAuth2 Bearer:
+//		- read:statuses
+//
+//	responses:
+//		'200':
+//			description: "The requested statuses."
+//			schema:
+//				type: array
+//				items:
+//					"$ref": "#/definitions/status"
+//		'400':
+//			schema:
+//				"$ref": "#/definitions/error"
+//			description: bad request
+//		'401':
+//			schema:
+//				"$ref": "#/definitions/error"
+//			description: unauthorized
+//		'403':
+//			schema:
+//				"$ref": "#/definitions/error"
+//			description: forbidden
+//		'406':
+//			schema:
+//				"$ref": "#/definitions/error"
+//			description: not acceptable
+//		'422':
+//			schema:
+//				"$ref": "#/definitions/error"
+//			description: unprocessable entity
+//		'500':
+//			schema:
+//				"$ref": "#/definitions/error"
+//			description: internal server error
+func (m *Module) StatusesGETHandler(c *gin.Context) {
+	authed, errWithCode := apiutil.TokenAuth(c,
+		true, true, true, true,
+		apiutil.ScopeReadStatuses,
+	)
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	if _, errWithCode := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	ids := c.QueryArray("id[]")
+	if len(ids) == 0 {
+		ids = c.QueryArray("id")
+	}
+
+	if len(ids) == 0 {
+		const text = "no status ids specified"
+		errWithCode := gtserror.NewErrorBadRequest(errors.New(text), text)
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	} else if len(ids) > 100 {
+		const text = "too many status ids specified"
+		errWithCode := gtserror.NewErrorUnprocessableEntity(errors.New(text), text)
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	apiStatuses, errWithCode := m.processor.Status().GetMultiple(c.Request.Context(), authed.Account, ids)
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	apiutil.JSON(c, http.StatusOK, apiStatuses)
+}

internal/api/client/statuses/statusesget_test.go

@@ -0,0 +1,173 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package statuses_test
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"code.superseriousbusiness.org/gotosocial/internal/api/client/statuses"
+	apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
+	"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
+	"code.superseriousbusiness.org/gotosocial/internal/oauth"
+	"code.superseriousbusiness.org/gotosocial/testrig"
+	"github.com/stretchr/testify/suite"
+)
+
+type StatusesGetTestSuite struct {
+	StatusStandardTestSuite
+}
+
+func (suite *StatusesGetTestSuite) getStatuses(
+	requestingAccount *gtsmodel.Account,
+	token *gtsmodel.Token,
+	user *gtsmodel.User,
+	statusIDs []string,
+	expectedHTTPStatus int,
+	expectedBody string,
+) ([]apimodel.Status, error) {
+	recorder := httptest.NewRecorder()
+	ctx, _ := testrig.CreateGinTestContext(recorder, nil)
+
+	requestURL := testrig.URLMustParse("/api" + statuses.BasePath)
+	query := url.Values{}
+	for _, statusID := range statusIDs {
+		query.Add("id[]", statusID)
+	}
+
+	requestURL.RawQuery = query.Encode()
+	ctx.Request = httptest.NewRequest(http.MethodGet, requestURL.String(), nil)
+	ctx.Request.Header.Set("accept", "application/json")
+
+	if token != nil {
+		ctx.Set(oauth.SessionAuthorizedToken, oauth.DBTokenToToken(token))
+		ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+	}
+	if requestingAccount != nil {
+		ctx.Set(oauth.SessionAuthorizedAccount, requestingAccount)
+	}
+	if user != nil {
+		ctx.Set(oauth.SessionAuthorizedUser, user)
+	}
+
+	suite.statusModule.StatusesGETHandler(ctx)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+
+	body, err := io.ReadAll(result.Body)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal(expectedHTTPStatus, recorder.Code, string(body))
+
+	if expectedBody != "" {
+		suite.Equal(expectedBody, string(body))
+		return nil, nil
+	}
+
+	apiStatuses := make([]apimodel.Status, 0)
+	if err := json.Unmarshal(body, &apiStatuses); err != nil {
+		suite.FailNow(err.Error(), string(body))
+	}
+
+	return apiStatuses, nil
+}
+
+func (suite *StatusesGetTestSuite) TestGetStatusesAuthenticatedOK() {
+	apiStatuses, err := suite.getStatuses(
+		suite.testAccounts["local_account_1"],
+		suite.testTokens["local_account_1"],
+		suite.testUsers["local_account_1"],
+		[]string{
+			suite.testStatuses["admin_account_status_1"].ID,
+			suite.testStatuses["admin_account_status_2"].ID,
+			"01ZZZZZZZZZZZZZZZZZZZZZZZZ",
+		},
+		http.StatusOK,
+		"",
+	)
+	suite.NoError(err)
+	suite.Len(apiStatuses, 2)
+	suite.Equal(suite.testStatuses["admin_account_status_1"].ID, apiStatuses[0].ID)
+	suite.Equal(suite.testStatuses["admin_account_status_2"].ID, apiStatuses[1].ID)
+}
+
+func (suite *StatusesGetTestSuite) TestGetStatusesNoIDsBadRequest() {
+	_, err := suite.getStatuses(
+		suite.testAccounts["local_account_1"],
+		suite.testTokens["local_account_1"],
+		suite.testUsers["local_account_1"],
+		nil,
+		http.StatusBadRequest,
+		`{"error":"Bad Request: no status ids specified"}`,
+	)
+	suite.NoError(err)
+}
+
+func (suite *StatusesGetTestSuite) TestGetStatusesTooManyIDs() {
+	_, err := suite.getStatuses(
+		suite.testAccounts["local_account_1"],
+		suite.testTokens["local_account_1"],
+		suite.testUsers["local_account_1"],
+		make([]string, 101),
+		http.StatusUnprocessableEntity,
+		`{"error":"Unprocessable Entity: too many status ids specified"}`,
+	)
+	suite.NoError(err)
+}
+
+func (suite *StatusesGetTestSuite) TestGetStatusesInsufficientScope() {
+	_, err := suite.getStatuses(
+		suite.testAccounts["local_account_1"],
+		suite.testTokens["local_account_1_push_only"],
+		suite.testUsers["local_account_1"],
+		[]string{suite.testStatuses["admin_account_status_1"].ID},
+		http.StatusForbidden,
+		`{"error":"Forbidden: token has insufficient scope permission"}`,
+	)
+	suite.NoError(err)
+}
+
+func (suite *StatusesGetTestSuite) TestGetStatusesDeduplicatesAndKeepsOrder() {
+	apiStatuses, err := suite.getStatuses(
+		suite.testAccounts["local_account_1"],
+		suite.testTokens["local_account_1"],
+		suite.testUsers["local_account_1"],
+		[]string{
+			suite.testStatuses["admin_account_status_2"].ID,
+			suite.testStatuses["admin_account_status_1"].ID,
+			suite.testStatuses["admin_account_status_2"].ID,
+		},
+		http.StatusOK,
+		"",
+	)
+	suite.NoError(err)
+	suite.Len(apiStatuses, 2)
+	suite.Equal(suite.testStatuses["admin_account_status_2"].ID, apiStatuses[0].ID)
+	suite.Equal(suite.testStatuses["admin_account_status_1"].ID, apiStatuses[1].ID)
+}
+
+func TestStatusesGetTestSuite(t *testing.T) {
+	suite.Run(t, new(StatusesGetTestSuite))
+}

internal/cache/db.go

@@ -1582,6 +1582,9 @@ func (c *Caches) initStatus() {
 		s2.CreatedWithApplication = nil
 		s2.Edits = nil
 
+		// Zero non-db fields.
+		s2.Edited = false
+
 		return s2
 	}
 

internal/federation/dereferencing/account.go

@@ -54,10 +54,6 @@ func accountFresh(
 	account *gtsmodel.Account,
 	window *FreshnessWindow,
 ) bool {
-	if window == nil {
-		window = DefaultAccountFreshness
-	}
-
 	if account.IsLocal() {
 		// Can't refresh
 		// local accounts.
@@ -77,12 +73,17 @@ func accountFresh(
 		return true
 	}
 
+	if window == nil {
+		// If no window given, fallback
+		// to default account freshness.
+		window = &DefaultAccountFreshness
+	}
+
 	// Moment when the account is
 	// considered stale according to
 	// desired freshness window.
 	staleAt := account.FetchedAt.Add(
-		time.Duration(*window),
-	)
+		time.Duration(*window))
 
 	// It's still fresh if the time now
 	// is not past the point of staleness.
@@ -143,15 +144,11 @@ func (d *Dereferencer) getAccountByURI(
 	uri *url.URL,
 	tryURL bool,
 ) (*gtsmodel.Account, ap.Accountable, error) {
-	var (
-		account *gtsmodel.Account
-		uriStr  = uri.String()
-		err     error
-	)
+	var uriStr = uri.String()
 
 	// Search the database for existing account with URI.
 	// URI is unique so if we get a hit it's that account for sure.
-	account, err = d.state.DB.GetAccountByURI(
+	account, err := d.state.DB.GetAccountByURI(
 		gtscontext.SetBarebones(ctx),
 		uriStr,
 	)
@@ -459,9 +456,9 @@ func (d *Dereferencer) enrichAccountSafely(
 	// By default use account.URI
 	// as the per-URI deref lock.
 	var uriStr string
-	if account.URI != "" {
-		uriStr = account.URI
-	} else {
+	uriStr = account.URI
+	if uriStr == "" {
+
 		// No URI is set yet, instead generate a faux-one from user+domain.
 		uriStr = "https://" + account.Domain + "/users/" + account.Username
 	}
@@ -506,7 +503,12 @@ func (d *Dereferencer) enrichAccountSafely(
 	// we're done.
 	unlock()
 
-	if errors.Is(err, db.ErrAlreadyExists) {
+	switch {
+	case err == nil:
+		// Pass account to dereferencer hook.
+		d.onAccountDereference(ctx, latest)
+
+	case errors.Is(err, db.ErrAlreadyExists):
 		// Ensure AP model isn't set,
 		// otherwise this indicates WE
 		// enriched the account.

internal/federation/dereferencing/announce.go

@@ -34,18 +34,17 @@ import (
 // The wrapper is then returned to the caller.
 //
 // The provided boost wrapper status must have BoostOfURI set.
-//
-// The returned boolean indicates whether the target of the boost
-// is new (to us), ie., it has not been dereferenced or seen before.
 func (d *Dereferencer) EnrichAnnounce(
 	ctx context.Context,
 	boost *gtsmodel.Status,
 	requestUser string,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
-) (*gtsmodel.Status, bool, error) {
+) (
+	*gtsmodel.Status,
+	error,
+) {
 	if boost.BoostOfURIStr == "" {
 		// We can't do anything.
-		return nil, false, gtserror.Newf("no URI to dereference")
+		return nil, gtserror.Newf("no URI to dereference")
 	}
 
 	// Take relevant URIs.
@@ -56,20 +55,15 @@ func (d *Dereferencer) EnrichAnnounce(
 	//
 	// GetStatusByURI accounts for domain blocks and local
 	// statuses, and also updates targetURI in case of redirects.
-	target, _, targetIsNew, err := d.GetStatusByURI(
-		ctx,
-		requestUser,
-		targetURI,
-		newThreadEntryCallback,
-	)
+	target, _, err := d.GetStatusByURI(ctx, requestUser, targetURI)
 	if err != nil {
-		return nil, false, gtserror.Newf("error fetching boost target %s: %w", targetURIStr, err)
+		return nil, gtserror.Newf("error fetching boost target %s: %w", targetURIStr, err)
 	}
 
 	if target.BoostOfID != "" {
 		// Ensure that the target is not a boost (should not be possible).
 		err := gtserror.Newf("target status %s is a boost", targetURIStr)
-		return nil, false, err
+		return nil, err
 	}
 
 	// Set boost of URIs again in case the
@@ -92,15 +86,16 @@ func (d *Dereferencer) EnrichAnnounce(
 	boost.Flags.SetFederated(target.Flags.Federated())
 
 	// Ensure this Announce is permitted by the Announcee.
-	permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost, true)
+	permit, err := d.isPermittedStatus(ctx, requestUser,
+		nil, boost, true)
 	if err != nil {
-		return nil, false, gtserror.Newf("error checking permitted status %s: %w", boost.URI, err)
+		return nil, gtserror.Newf("error checking permitted status %s: %w", boost.URI, err)
 	}
 
 	if !permit {
 		// Return a checkable error type that can be ignored.
 		err := gtserror.Newf("dropping unpermitted status: %s", boost.URI)
-		return nil, false, gtserror.SetNotPermitted(err)
+		return nil, gtserror.SetNotPermitted(err)
 	}
 
 	// Generate an ID for the boost wrapper status.
@@ -118,15 +113,18 @@ func (d *Dereferencer) EnrichAnnounce(
 		// in a call to db.Put(Status). Look again in DB by URI.
 		boost, err = d.state.DB.GetStatusByURI(ctx, uri)
 		if err != nil {
-			return nil, false, gtserror.Newf(
+			return nil, gtserror.Newf(
 				"error getting boost wrapper status %s from database after race: %w",
 				uri, err,
 			)
 		}
 
 	default: // Proper database error.
-		return nil, false, gtserror.Newf("db error inserting status: %w", err)
+		return nil, gtserror.Newf("db error inserting status: %w", err)
 	}
 
-	return boost, targetIsNew, err
+	// Pass status to its dereferencer hook.
+	d.onStatusDereference(ctx, boost, true)
+
+	return boost, err
 }

internal/federation/dereferencing/dereferencer.go

@@ -18,18 +18,20 @@
 package dereferencing
 
 import (
+	"context"
 	"net/url"
 	"sync"
 	"time"
 
+	"code.superseriousbusiness.org/gopkg/log"
 	"code.superseriousbusiness.org/gotosocial/internal/filter/interaction"
 	"code.superseriousbusiness.org/gotosocial/internal/filter/relay"
 	"code.superseriousbusiness.org/gotosocial/internal/filter/visibility"
+	"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
 	"code.superseriousbusiness.org/gotosocial/internal/media"
 	"code.superseriousbusiness.org/gotosocial/internal/state"
 	"code.superseriousbusiness.org/gotosocial/internal/transport"
 	"code.superseriousbusiness.org/gotosocial/internal/typeutils"
-	"code.superseriousbusiness.org/gotosocial/internal/util"
 )
 
 // FreshnessWindow represents a duration in which a
@@ -42,20 +44,20 @@ import (
 // according to DefaultAccountFreshness, but not according
 // to Fresh, which would indicate that the Account requires
 // refreshing from remote.
-type FreshnessWindow time.Duration
+type FreshnessWindow = time.Duration
 
 var (
 	// 6 hours.
 	//
 	// Default window for doing a
 	// fresh dereference of an Account.
-	DefaultAccountFreshness = util.Ptr(FreshnessWindow(6 * time.Hour))
+	DefaultAccountFreshness = 6 * time.Hour
 
 	// 2 hours.
 	//
 	// Default window for doing a
 	// fresh dereference of a Status.
-	DefaultStatusFreshness = util.Ptr(FreshnessWindow(2 * time.Hour))
+	DefaultStatusFreshness = 2 * time.Hour
 
 	// 5 minutes.
 	//
@@ -65,17 +67,16 @@ var (
 	//
 	// This is tuned to be quite fresh without
 	// causing loads of dereferencing calls.
-	Fresh = util.Ptr(FreshnessWindow(5 * time.Minute))
+	Fresh = 5 * time.Minute
 
-	// 5 seconds.
+	// Immediate.
 	//
-	// Freshest is useful when you want an
-	// immediately up to date model of something
-	// that's even fresher than Fresh.
-	//
-	// Be careful using this one; it can cause
-	// lots of unnecessary traffic if used unwisely.
-	Freshest = util.Ptr(FreshnessWindow(5 * time.Second))
+	// This essentially always allows a refresh,
+	// and should only be used if a model update
+	// was pushed (federated) to the server,
+	// i.e. no model dereference is required.
+	// Otherwise it could DoS the model's host.
+	Freshest = time.Nanosecond
 )
 
 // Dereferencer wraps logic and functionality for doing dereferencing
@@ -89,6 +90,29 @@ type Dereferencer struct {
 	intFilter           *interaction.Filter
 	relayFilter         *relay.Filter
 
+	// OnAccountDereference is a hook that gets called on dereference of an account model.
+	// It is plumbed-in to the dereferencer but unused. In time it would be nice to add a
+	// new websocket API message type "update.account" that sends account model updates.
+	OnAccountDereference func(ctx context.Context, account *gtsmodel.Account) error
+
+	// OnStatusDereference is a hook that gets called on dereference of a status
+	// model, also indicating whether it was new to us at the time of dereference.
+	// This can be used to handle streaming and notifying of status create / update events.
+	//
+	// see: ./internal/surfacing/surfacing.go
+	OnStatusDereference func(ctx context.Context, status *gtsmodel.Status, isNew bool) error
+
+	// OnMediaDereference is a hook that gets called on dereference of a media attachment.
+	// This can be used to handle streaming of updated status models when media finishes processing.
+	//
+	// see: ./internal/surfacing/surfacing.go
+	OnMediaDereference func(ctx context.Context, media *gtsmodel.MediaAttachment) error
+
+	// OnEmojiDereference is a hook that gets called on dereference of an emoji attachment.
+	// It is plumbed-in to the dereferencer but unused. In time it would be nice to add a
+	// new websocket API message type "update.emoji" that sends emoji updates when finished processing.
+	OnEmojiDereference func(ctx context.Context, emoji *gtsmodel.Emoji) error
+
 	// in-progress dereferencing media / emoji
 	derefMedia    keyedList[*media.ProcessingMedia]
 	derefMediaMu  sync.Mutex
@@ -128,3 +152,35 @@ func NewDereferencer(
 		handshakes:          make(map[string][]*url.URL),
 	}
 }
+
+func (d *Dereferencer) onAccountDereference(ctx context.Context, account *gtsmodel.Account) {
+	if d.OnAccountDereference != nil {
+		if err := d.OnAccountDereference(ctx, account); err != nil {
+			log.Errorf(ctx, "error dereferencing account: %w", err)
+		}
+	}
+}
+
+func (d *Dereferencer) onStatusDereference(ctx context.Context, status *gtsmodel.Status, isNew bool) {
+	if d.OnStatusDereference != nil {
+		if err := d.OnStatusDereference(ctx, status, isNew); err != nil {
+			log.Errorf(ctx, "error dereferencing status: %w", err)
+		}
+	}
+}
+
+func (d *Dereferencer) onMediaDereference(ctx context.Context, media *gtsmodel.MediaAttachment) {
+	if d.OnMediaDereference != nil {
+		if err := d.OnMediaDereference(ctx, media); err != nil {
+			log.Errorf(ctx, "error dereferencing media: %w", err)
+		}
+	}
+}
+
+func (d *Dereferencer) onEmojiDereference(ctx context.Context, emoji *gtsmodel.Emoji) {
+	if d.OnEmojiDereference != nil {
+		if err := d.OnEmojiDereference(ctx, emoji); err != nil {
+			log.Errorf(ctx, "error dereferencing emoji: %w", err)
+		}
+	}
+}

internal/federation/dereferencing/emoji.go

@@ -379,10 +379,13 @@ func (d *Dereferencer) processEmojiSafely(
 			}
 
 			// Perform emoji load operation.
-			_, err = processing.Load(ctx)
+			emoji, err = processing.Load(ctx)
 			if err != nil {
 				log.Errorf(ctx, "error loading emoji %s: %v", shortcodeDomain, err)
 			}
+
+			// Pass to its dereferencer hook.
+			d.onEmojiDereference(ctx, emoji)
 		})
 	} else {
 		if !existing {
@@ -405,6 +408,9 @@ func (d *Dereferencer) processEmojiSafely(
 			// which can determine if loading error should allow remaining placeholder.
 			err = gtserror.Newf("error loading emoji %s: %w", shortcodeDomain, err)
 		}
+
+		// Pass to its dereferencer hook.
+		d.onEmojiDereference(ctx, emoji)
 	}
 
 	return

internal/federation/dereferencing/media.go

@@ -333,10 +333,13 @@ func (d *Dereferencer) processMediaSafely(
 			}
 
 			// Perform media load operation.
-			_, err = processing.Load(ctx)
+			attach, err = processing.Load(ctx)
 			if err != nil {
 				log.Errorf(ctx, "error loading media %s: %v", remoteURL, err)
 			}
+
+			// Pass to its dereferencer hook.
+			d.onMediaDereference(ctx, attach)
 		})
 	} else {
 		if !existing {
@@ -359,6 +362,9 @@ func (d *Dereferencer) processMediaSafely(
 			// which can determine if loading error should allow remaining placeholder.
 			err = gtserror.Newf("error loading media %s: %w", remoteURL, err)
 		}
+
+		// Pass to its dereferencer hook.
+		d.onMediaDereference(ctx, attach)
 	}
 
 	return

internal/federation/dereferencing/status.go

@@ -38,28 +38,26 @@ import (
 // first checking the database. In the case of a newly-met
 // remote model, or a remote model whose 'last_fetched' date
 // is beyond a certain interval, the status will be dereferenced.
+// Upon dereferencing the status model, (and any subsequent models
+// discovered during thread iteration), will be passed to the
+// OnStatusDereference() hook to handle timelining, streaming and
+// notification events. THOUGH DO NOTE THAT THIS WILL BE SKIPPED
+// IF THE STATUS IS STILL PENDING APPROVAL.
 //
 // A returned AP statusable indicates the status was dereferenced.
-// The returned bool indicates whether the status was new (to us).
 //
-// In the case of dereferencing, some low-priority status info
-// will be enqueued for asynchronous fetching, e.g. dereferencing
-// the status thread.
-//
-// If newThreadEntryCallback is set, it will be called for each
-// newly-discovered status in the thread other than the requested
-// status itself.
+// In the case of dereferencing, some low-priority status info will be
+// enqueued for asynchronous fetching, e.g. dereferencing status thread.
 func (d *Dereferencer) GetStatusByURI(
 	ctx context.Context,
 	requestUser string,
 	uri *url.URL,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
 ) (
 	status *gtsmodel.Status,
 	statusable ap.Statusable,
-	isNew bool,
 	err error,
 ) {
+	var isNew bool
 
 	// Fetch and dereference / update status if necessary.
 	status, statusable, isNew, err = d.getStatusByURI(ctx,
@@ -71,7 +69,7 @@ func (d *Dereferencer) GetStatusByURI(
 		if status == nil {
 			// err with no existing
 			// status for fallback.
-			return nil, nil, false, err
+			return nil, nil, err
 		}
 
 		log.Errorf(ctx, "error updating status %s: %v", uri, err)
@@ -85,11 +83,10 @@ func (d *Dereferencer) GetStatusByURI(
 			status,
 			statusable,
 			isNew,
-			newThreadEntryCallback,
 		)
 	}
 
-	return status, statusable, isNew, nil
+	return status, statusable, nil
 }
 
 // RefreshStatus is functionally equivalent to GetStatusByURI(),
@@ -98,17 +95,12 @@ func (d *Dereferencer) GetStatusByURI(
 //
 // A returned AP statusable indicates the status was dereferenced.
 // The returned bool indicates whether the status was new (to us).
-//
-// If newThreadEntryCallback is set, it will be called for each
-// newly-discovered status in the thread other than the requested
-// status itself.
 func (d *Dereferencer) RefreshStatus(
 	ctx context.Context,
 	requestUser string,
 	status *gtsmodel.Status,
 	statusable ap.Statusable,
 	window *FreshnessWindow,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
 ) (
 	latest *gtsmodel.Status,
 	latestStatusable ap.Statusable,
@@ -127,9 +119,9 @@ func (d *Dereferencer) RefreshStatus(
 		return nil, nil, gtserror.Newf("invalid status uri %q: %w", status.URI, err)
 	}
 
-	// Try to update + dereference
-	// the passed status model.
 	var isNew bool
+
+	// Try to update and dereference the passed status model.
 	latest, latestStatusable, isNew, err = d.enrichAndStoreStatusSafely(ctx,
 		requestUser,
 		uri,
@@ -145,13 +137,66 @@ func (d *Dereferencer) RefreshStatus(
 			latest,
 			latestStatusable,
 			isNew,
-			newThreadEntryCallback,
 		)
 	}
 
 	return latest, latestStatusable, err
 }
 
+// RefreshStatusAsync is functionally equivalent to callling RefreshStatus()
+// yourself within a dereferencer worker function, except that it performs an
+// optimized hand-off operation by performing freshness and validity checks
+// synchronously beforehand. This prevents handing spurious tasks to the worker.
+func (d *Dereferencer) RefreshStatusAsync(
+	ctx context.Context,
+	requestUser string,
+	status *gtsmodel.Status,
+	statusable ap.Statusable,
+	window *FreshnessWindow,
+) {
+	// If no incoming data is provided,
+	// check whether status needs update.
+	if statusable == nil &&
+		statusFresh(status, window) {
+		return
+	}
+
+	// Parse the URI from status.
+	uri, err := url.Parse(status.URI)
+	if err != nil {
+		log.Errorf(ctx, "invalid status uri %q: %v", status.URI, err)
+		return
+	}
+
+	// Enqueue a worker function to enrich this status model async.
+	d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) {
+		var isNew bool
+
+		// Try to update and dereference the passed status model.
+		latest, statusable, isNew, err := d.enrichAndStoreStatusSafely(ctx,
+			requestUser,
+			uri,
+			status,
+			statusable,
+		)
+		if err != nil {
+			log.Errorf(ctx, "error enriching remote status: %v", err)
+			return
+		}
+
+		if statusable != nil {
+			// Deref parents + children.
+			d.dereferenceThread(ctx,
+				requestUser,
+				uri,
+				latest,
+				statusable,
+				isNew,
+			)
+		}
+	})
+}
+
 /*
 	INTERNAL / UTIL FUNCTIONS HERE
 */
@@ -179,7 +224,7 @@ func statusFresh(
 	if window == nil {
 		// If no window given, fallback
 		// to default status freshness.
-		window = DefaultStatusFreshness
+		window = &DefaultStatusFreshness
 	}
 
 	// Moment when the status is
@@ -227,7 +272,7 @@ func (d *Dereferencer) getStatusByURI(
 			&gtsmodel.Status{URI: uriStr}, nil)
 	}
 
-	if statusFresh(status, DefaultStatusFreshness) {
+	if statusFresh(status, &DefaultStatusFreshness) {
 		// This is an existing status that is up-to-date,
 		// before returning ensure it is fully populated.
 		if err := d.state.DB.PopulateStatus(ctx, status); err != nil {
@@ -257,7 +302,12 @@ func (d *Dereferencer) enrichAndStoreStatusSafely(
 	uri *url.URL,
 	status *gtsmodel.Status,
 	statusable ap.Statusable,
-) (*gtsmodel.Status, ap.Statusable, bool, error) {
+) (
+	*gtsmodel.Status,
+	ap.Statusable,
+	bool,
+	error,
+) {
 	uriStr := status.URI
 
 	// Acquire per-URI deref lock, wraping unlock
@@ -268,6 +318,8 @@ func (d *Dereferencer) enrichAndStoreStatusSafely(
 	defer unlock()
 
 	if status.ID != "" {
+		var err error
+
 		// If ID was set it means we've stored this status before.
 		//
 		// We reload the existing status, just to ensure we have the
@@ -275,7 +327,6 @@ func (d *Dereferencer) enrichAndStoreStatusSafely(
 		// just input a change but we still have an old status copy.
 		//
 		// Note: returned status will be fully populated, required below.
-		var err error
 		status, err = d.state.DB.GetStatusByID(ctx, status.ID)
 		if err != nil {
 			return nil, nil, false, gtserror.Newf("error getting up-to-date existing status: %w", err)
@@ -333,7 +384,12 @@ func (d *Dereferencer) enrichAndStoreStatusSafely(
 	// we're done.
 	unlock()
 
-	if errors.Is(err, db.ErrAlreadyExists) {
+	switch {
+	case err == nil:
+		// Pass status to its dereferencer hook.
+		d.onStatusDereference(ctx, latest, isNew)
+
+	case errors.Is(err, db.ErrAlreadyExists):
 		// We leave 'isNew' set so that caller
 		// still dereferences parents, otherwise
 		// the version we pass back may not have
@@ -455,7 +511,7 @@ func (d *Dereferencer) enrichAndStoreStatus(
 	//
 	// If a remote has in the meantime retracted its approval,
 	// the next call to 'isPermittedStatus' will catch that.
-	if latestStatus.ApprovedByURI == "" && status.ApprovedByURI != "" {
+	if latestStatus.ApprovedByURI == "" {
 		latestStatus.ApprovedByURI = status.ApprovedByURI
 	}
 

internal/federation/dereferencing/status_common.go

@@ -39,16 +39,15 @@ func (d *Dereferencer) getStatusDBOnly(
 	ctx context.Context,
 	uriStr string,
 ) (*gtsmodel.Status, error) {
-	// Request a barebones object:
-	// status may be in the db but with
-	// related models not yet dereffed.
-	ctxBb := gtscontext.SetBarebones(ctx)
+	// For both queries request a barebones
+	// object, as it will be later populated
+	// in the enrichAndStoreSafely() function.
+	ctx = gtscontext.SetBarebones(ctx)
 
-	// Search the database for existing by URI.
-	status, err := d.state.DB.GetStatusByURI(ctxBb, uriStr)
+	// Search the database for existing status by URI.
+	status, err := d.state.DB.GetStatusByURI(ctx, uriStr)
 	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		err := gtserror.Newf("error checking database for status %s by uri: %w", uriStr, err)
-		return nil, err
+		return nil, gtserror.Newf("error checking database for status %s by uri: %w", uriStr, err)
 	}
 
 	if status != nil {
@@ -57,11 +56,10 @@ func (d *Dereferencer) getStatusDBOnly(
 		return status, nil
 	}
 
-	// Else, search database for existing by URL.
-	status, err = d.state.DB.GetStatusByURL(ctxBb, uriStr)
+	// Else, search database for existing status by URL.
+	status, err = d.state.DB.GetStatusByURL(ctx, uriStr)
 	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		err := gtserror.Newf("error checking database for status %s by url: %w", uriStr, err)
-		return nil, err
+		return nil, gtserror.Newf("error checking database for status %s by url: %w", uriStr, err)
 	}
 
 	// Return maybe status.
@@ -155,9 +153,8 @@ func (d *Dereferencer) retrieveStatusable(
 		// There's not a match, so the remote is doing
 		// something weird. Gather URI strings we would
 		// have accepted into nice slice for logging.
-		var okURIStrs []string
-		okURIStrs = xslices.Gather(
-			okURIStrs,
+		okURIStrs := xslices.Gather(
+			nil,
 			okURIs,
 			func(u *url.URL) string {
 				return u.String()

internal/federation/dereferencing/status_peripherals.go

@@ -171,14 +171,11 @@ func (d *Dereferencer) fetchStatusMentions(
 	status.MentionIDs = make([]string, len(status.Mentions))
 
 	for i := range status.Mentions {
-		var (
-			mention       = status.Mentions[i]
-			alreadyExists bool
-		)
+		mention := status.Mentions[i]
 
 		// Search existing status + db for a mention already stored,
 		// else ensure new mention's target account is populated.
-		mention, alreadyExists, err = d.newOrExistingMention(ctx,
+		mention, alreadyExists, err := d.newOrExistingMention(ctx,
 			requestUser,
 			existing,
 			mention,
@@ -728,6 +725,10 @@ func (d *Dereferencer) handleStatusEdit(
 
 		// Add edit to list of cols.
 		cols = append(cols, "edits")
+
+		// Mark the status as edited for
+		// appropriate handling in hooks.
+		status.Edited = true
 	}
 
 	if !existing.EditedAt.Equal(status.EditedAt) {

internal/federation/dereferencing/status_relayed.go

@@ -56,7 +56,6 @@ func (d *Dereferencer) GetRelayedStatus(
 	instanceAcct *gtsmodel.Account,
 	relayAcct *gtsmodel.Account,
 	uri *url.URL,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
 ) (*gtsmodel.Status, error) {
 	// Check whether this status URI is a blocked domain / subdomain.
 	if blocked, err := d.state.DB.IsDomainBlocked(ctx, uri.Host); err != nil {
@@ -166,7 +165,6 @@ func (d *Dereferencer) GetRelayedStatus(
 		status,
 		statusable,
 		true, // isNew = true
-		newThreadEntryCallback,
 	)
 
 	return status, nil
@@ -177,7 +175,6 @@ func (d *Dereferencer) GetRelayedAnnounce(
 	instanceAcct *gtsmodel.Account,
 	relayAcct *gtsmodel.Account,
 	boostWrapper *gtsmodel.Status,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
 ) (*gtsmodel.Status, error) {
 	// Check whether boosted status URI
 	// is a blocked domain / subdomain.
@@ -315,7 +312,6 @@ func (d *Dereferencer) GetRelayedAnnounce(
 		status,
 		statusable,
 		true, // isNew = true
-		newThreadEntryCallback,
 	)
 
 	return status, nil

internal/federation/dereferencing/status_test.go

@@ -43,7 +43,7 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839")
-	status, _, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL, nil)
+	status, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL)
 	suite.NoError(err)
 	suite.NotNil(status)
 
@@ -81,7 +81,7 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV")
-	status, _, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL, nil)
+	status, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL)
 	suite.NoError(err)
 	suite.NotNil(status)
 
@@ -130,7 +130,7 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithTag() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7")
-	status, _, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL, nil)
+	status, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL)
 	suite.NoError(err)
 	suite.NotNil(status)
 
@@ -178,7 +178,7 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithImageAndNoContent() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	statusURL := testrig.URLMustParse("https://turnip.farm/users/turniplover6969/statuses/70c53e54-3146-42d5-a630-83c8b6c7c042")
-	status, _, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL, nil)
+	status, _, err := suite.dereferencer.GetStatusByURI(suite.T().Context(), fetchingAccount.Username, statusURL)
 	suite.NoError(err)
 	suite.NotNil(status)
 
@@ -231,11 +231,10 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithNonMatchingURI() {
 	suite.client.TestRemoteStatuses[remoteAltURI] = remoteStatus
 
 	// Attempt to fetch account at alternative URI, it should fail!
-	fetchedStatus, _, _, err := suite.dereferencer.GetStatusByURI(
+	fetchedStatus, _, err := suite.dereferencer.GetStatusByURI(
 		suite.T().Context(),
 		fetchingAccount.Username,
 		testrig.URLMustParse(remoteAltURI),
-		nil,
 	)
 	expectErrStr := fmt.Sprintf(
 		"retrieveStatusable: http URI %s does not match dereferenced statusable id or url(s) [%s %s]",
@@ -257,10 +256,9 @@ func (suite *StatusTestSuite) TestDereferencerRefreshStatusUpdated() {
 	testStatusable := suite.client.TestRemoteStatuses[testURIStr]
 
 	// Fetch the remote status first to load it into instance.
-	testStatus, statusable, _, err := suite.dereferencer.GetStatusByURI(ctx,
+	testStatus, statusable, err := suite.dereferencer.GetStatusByURI(ctx,
 		fetchingAccount.Username,
 		testURI,
-		nil,
 	)
 	suite.NotNil(statusable)
 	suite.NoError(err)
@@ -309,7 +307,6 @@ func (suite *StatusTestSuite) TestDereferencerRefreshStatusUpdated() {
 			testStatus,
 			nil, // NOTE: can provide testStatusable here to test as being received (not deref'd)
 			instantFreshness,
-			nil,
 		)
 		suite.NotNil(statusable)
 		suite.NoError(err)
@@ -364,10 +361,9 @@ func (suite *StatusTestSuite) TestDereferencerRefreshStatusRace() {
 	testStatusable := suite.client.TestRemoteStatuses[testURIStr]
 
 	// Fetch the remote status first to load it into instance.
-	testStatus, statusable, _, err := suite.dereferencer.GetStatusByURI(ctx,
+	testStatus, statusable, err := suite.dereferencer.GetStatusByURI(ctx,
 		fetchingAccount.Username,
 		testURI,
-		nil,
 	)
 	suite.NotNil(statusable)
 	suite.NoError(err)
@@ -394,7 +390,6 @@ func (suite *StatusTestSuite) TestDereferencerRefreshStatusRace() {
 		testStatus,
 		testStatusable,
 		instantFreshness,
-		nil,
 	)
 	suite.NotNil(statusable)
 	suite.NoError(err)
@@ -444,7 +439,6 @@ func (suite *StatusTestSuite) TestDereferencerRefreshStatusRace() {
 		beforeEdit,
 		testStatusable,
 		instantFreshness,
-		nil,
 	)
 	suite.NotNil(statusable)
 	suite.NoError(err)

internal/federation/dereferencing/thread.go

@@ -33,7 +33,7 @@ import (
 
 // maxIter defines how many iterations of descendants or
 // ancesters we are willing to follow before returning error.
-const maxIter = 512
+const maxIter = 1024
 
 // dereferenceThread handles dereferencing status thread after
 // fetch. Passing off appropriate parts to be enqueued for async
@@ -48,30 +48,29 @@ func (d *Dereferencer) dereferenceThread(
 	status *gtsmodel.Status,
 	statusable ap.Statusable,
 	isNew bool,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
 ) {
 	if isNew {
 		// This is a new status that we need the ancestors of in
 		// order to determine visibility. Perform the initial part
 		// of thread dereferencing, i.e. parents, synchronously.
-		err := d.dereferenceStatusAncestors(ctx, requestUser, status, newThreadEntryCallback)
+		err := d.dereferenceStatusAncestors(ctx, requestUser, status)
 		if err != nil {
 			log.Error(ctx, err)
 		}
 
 		// Enqueue dereferencing remaining status thread, (children), asychronously .
 		d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) {
-			if err := d.dereferenceStatusDescendants(ctx, requestUser, uri, statusable, newThreadEntryCallback); err != nil {
+			if err := d.dereferenceStatusDescendants(ctx, requestUser, uri, statusable); err != nil {
 				log.Error(ctx, err)
 			}
 		})
 	} else {
 		// This is an existing status, dereference the WHOLE thread asynchronously.
 		d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) {
-			if err := d.dereferenceStatusAncestors(ctx, requestUser, status, newThreadEntryCallback); err != nil {
+			if err := d.dereferenceStatusAncestors(ctx, requestUser, status); err != nil {
 				log.Error(ctx, err)
 			}
-			if err := d.dereferenceStatusDescendants(ctx, requestUser, uri, statusable, newThreadEntryCallback); err != nil {
+			if err := d.dereferenceStatusDescendants(ctx, requestUser, uri, statusable); err != nil {
 				log.Error(ctx, err)
 			}
 		})
@@ -88,7 +87,6 @@ func (d *Dereferencer) dereferenceStatusAncestors(
 	ctx context.Context,
 	username string,
 	status *gtsmodel.Status,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
 ) error {
 	// Start log entry with fields
 	l := log.WithContext(ctx).
@@ -134,7 +132,7 @@ func (d *Dereferencer) dereferenceStatusAncestors(
 
 		// Fetch parent status by current's reply URI, this handles
 		// case of existing (updating if necessary) or a new status.
-		parent, _, isNew, err := d.getStatusByURI(ctx, username, uri)
+		parent, _, _, err := d.getStatusByURI(ctx, username, uri)
 
 		// Check for a returned HTTP code via error.
 		switch code := gtserror.StatusCode(err); {
@@ -251,14 +249,6 @@ func (d *Dereferencer) dereferenceStatusAncestors(
 			}
 		}
 
-		// If parent is a brand new status (to us) and
-		// newThreadEntryCallback is defined, call it.
-		if isNew && newThreadEntryCallback != nil {
-			if err := newThreadEntryCallback(ctx, parent); err != nil {
-				l.Errorf("error during newThreadEntryCallback for status %s: %v", parent.URI, err)
-			}
-		}
-
 		// Set next parent to use.
 		current.InReplyTo = parent
 		current = current.InReplyTo
@@ -278,7 +268,6 @@ func (d *Dereferencer) dereferenceStatusDescendants(
 	username string,
 	statusIRI *url.URL,
 	parent ap.Statusable,
-	newThreadEntryCallback func(context.Context, *gtsmodel.Status) error,
 ) error {
 	statusIRIStr := statusIRI.String()
 
@@ -386,7 +375,7 @@ stackLoop:
 				//   - refetching recently fetched statuses (recursion!)
 				//   - remote domain is blocked (will return unretrievable)
 				//   - any http type error for a new status returns unretrievable
-				status, statusable, isNew, err := d.getStatusByURI(ctx, username, itemIRI)
+				_, statusable, _, err := d.getStatusByURI(ctx, username, itemIRI)
 				switch {
 				case err == nil:
 					// No problem!
@@ -410,8 +399,7 @@ stackLoop:
 					continue itemLoop
 
 				default:
-					// Something else went wrong,
-					// log this at error level.
+					// Something else went wrong, log this at error level.
 					l.Errorf("error dereferencing remote status %s: %v", itemIRI, err)
 					continue itemLoop
 				}
@@ -425,14 +413,6 @@ stackLoop:
 					continue itemLoop
 				}
 
-				// If child is a brand new status (to us) and
-				// newThreadEntryCallback is defined, call it.
-				if isNew && newThreadEntryCallback != nil {
-					if err := newThreadEntryCallback(ctx, status); err != nil {
-						l.Errorf("error during newThreadEntryCallback for status %s: %v", itemIRI, err)
-					}
-				}
-
 				// Extract any attached collection + ID URI from status.
 				page, pageURI := getAttachedStatusCollectionPage(statusable)
 				if page == nil {

internal/federation/federatingdb/accept.go

@@ -867,8 +867,8 @@ func (f *DB) acceptPoliteReplyRequest(
 	partial.intReq.AcceptedAt = time.Now()
 	partial.intReq.AuthorizationURI = authURIStr
 	partial.intReq.ResponseURI = acceptID.String()
-	if err := f.state.DB.UpdateInteractionRequest(
-		ctx, partial.intReq,
+	if err := f.state.DB.UpdateInteractionRequest(ctx,
+		partial.intReq,
 		"accepted_at",
 		"authorization_uri",
 		"response_uri",
@@ -878,8 +878,8 @@ func (f *DB) acceptPoliteReplyRequest(
 
 	reply.ApprovedByURI = authURIStr
 	reply.Flags.SetPendingApproval(false)
-	if err := f.state.DB.UpdateStatus(
-		ctx, reply,
+	if err := f.state.DB.UpdateStatus(ctx,
+		reply,
 		"approved_by_uri",
 		"flags",
 	); err != nil {

internal/gtsmodel/status.go

@@ -110,8 +110,12 @@ type Status struct {
 	// IDs of status edits for this status, ordered from
 	// smallest (oldest) -> largest (newest) ID. Edits of
 	// this status, ordered from oldest -> newest edit.
+	//
+	// Edited in particular is an ephemeral field
+	// only set on statuses that were just edited.
 	EditIDs []string      `bun:"edits,array"`
 	Edits   []*StatusEdit `bun:"-"`
+	Edited  bool          `bun:"-"`
 
 	// ID of the poll attached to this status,
 	// and the Poll that corresponds to pollID.

internal/processing/account/move.go

@@ -153,7 +153,7 @@ func (p *Processor) MoveSelf(
 			originAcct.Username,
 			targetAcct,
 			targetAcctable,
-			dereferencing.Freshest,
+			&dereferencing.Freshest,
 		)
 		if err != nil {
 			const text = "error dereferencing moved_to_uri"

internal/processing/common/status.go

@@ -244,11 +244,6 @@ func (p *Processor) getTargetStatusBy(
 			target,
 			nil,
 			window,
-
-			// Pass callback to insert
-			// other statuses in thread
-			// into timelines (as appropriate).
-			p.surfacer.TimelineAndNotifyStatus,
 		)
 		if err != nil {
 			log.Errorf(ctx, "error refreshing target %s: %v", target.URI, err)
@@ -322,30 +317,35 @@ func (p *Processor) GetVisibleAPIStatuses(
 			continue
 		}
 
-		// Check whether this status is muted by requesting account.
-		muted, err := p.muteFilter.StatusMuted(ctx, requester, status)
-		if err != nil {
-			log.Errorf(ctx, "error checking mute: %v", err)
-			continue
-		}
+		var filtered []apimodel.FilterResult
+		if filterCtx != 0 {
+			var hide bool
 
-		if muted {
-			continue
-		}
+			// Check whether this status is muted by requesting account.
+			muted, err := p.muteFilter.StatusMuted(ctx, requester, status)
+			if err != nil {
+				log.Errorf(ctx, "error checking mute: %v", err)
+				continue
+			}
 
-		// Check whether status is filtered in context by requesting account.
-		filtered, hide, err := p.statusFilter.StatusFilterResultsInContext(ctx,
-			requester,
-			status,
-			filterCtx,
-		)
-		if err != nil {
-			l.Errorf("error filtering: %v", err)
-			continue
-		}
+			if muted {
+				continue
+			}
 
-		if hide {
-			continue
+			// Check whether status is filtered in context by requesting account.
+			filtered, hide, err = p.statusFilter.StatusFilterResultsInContext(ctx,
+				requester,
+				status,
+				filterCtx,
+			)
+			if err != nil {
+				l.Errorf("error filtering: %v", err)
+				continue
+			}
+
+			if hide {
+				continue
+			}
 		}
 
 		// Not muted or "hide" filtered. Convert to API status.

internal/processing/processor.go

@@ -227,7 +227,7 @@ func NewProcessor(
 		parseMentionFunc: parseMentionFunc,
 	}
 
-	// Instantiate sub processors used by other sub-processors.
+	// Instantiate common sub-processors used by others.
 	processor.stream = stream.New(state, oauthServer)
 	processor.conversations = conversations.New(state, converter, visFilter, muteFilter, statusFilter)
 	surfacer := surfacing.New(state, converter, federator, &processor.stream, visFilter, muteFilter, statusFilter, emailSender, webPushSender, &processor.conversations)

internal/processing/search/get.go

@@ -626,27 +626,11 @@ func (p *Processor) statusByURI(
 	if resolve {
 		// We're allowed to resolve, leave the
 		// rest up to the dereferencer functions.
-		status, _, isNew, err := p.federator.GetStatusByURI(
+		status, _, err := p.federator.GetStatusByURI(
 			gtscontext.SetFastFail(ctx),
 			requestingAccount.Username,
 			uri,
-			// Pass callback to insert
-			// other statuses in thread
-			// into timelines (as appropriate).
-			p.surfacer.TimelineAndNotifyStatus,
 		)
-
-		// If the status is successfully and newly
-		// dereferenced, put it in timelines
-		// (as appropriate) before returning.
-		if err == nil && status != nil && isNew {
-			if err := p.surfacer.TimelineAndNotifyStatus(ctx, status); err != nil {
-				// Not a deal breaker but
-				// definitely error log this.
-				log.Errorf(ctx, "error timelining and notifying status after search: %v", err)
-			}
-		}
-
 		return status, err
 	}
 

internal/processing/status/debugvisibility.go

@@ -57,10 +57,9 @@ func (p *Processor) DebugVisibilityGet(ctx context.Context, requester *gtsmodel.
 	}
 
 	// Now we know we've been provided a valid URI, try fetch status.
-	status, _, _, err := p.federator.Dereferencer.GetStatusByURI(ctx,
+	status, _, err := p.federator.Dereferencer.GetStatusByURI(ctx,
 		requester.Username,
 		uri,
-		nil,
 	)
 	if err != nil {
 		log.Errorf(ctx, "error fetching status %s: %v", uri, err)

internal/processing/status/edit.go

@@ -334,6 +334,11 @@ func (p *Processor) Edit(
 		status.Poll = nil
 	}
 
+	// Set ephemeral edited flag
+	// so appropriate surfacing
+	// logic is followed for edit.
+	status.Edited = true
+
 	// Finally update the existing status model in the database.
 	if err := p.state.DB.UpdateStatus(ctx, status, cols...); err != nil {
 		err := gtserror.Newf("error updating status in db: %w", err)

internal/processing/status/get.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 
+	"code.superseriousbusiness.org/gopkg/xslices"
 	apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
 	"code.superseriousbusiness.org/gotosocial/internal/gtserror"
 	"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
@@ -40,6 +41,41 @@ func (p *Processor) Get(ctx context.Context, requester *gtsmodel.Account, status
 	return p.c.GetAPIStatus(ctx, requester, target)
 }
 
+// GetMultiple gets the given statuses with the same semantics as Get. Missing or invisible statuses are omitted.
+func (p *Processor) GetMultiple(ctx context.Context, requester *gtsmodel.Account, statusIDs []string) ([]apimodel.Status, gtserror.WithCode) {
+
+	// Without auth, just
+	// return equivalent of
+	// 404 not found for all.
+	if requester == nil {
+		return nil, nil
+	}
+
+	// Ensure we've only got unique statuses.
+	statusIDs = xslices.Deduplicate(statusIDs)
+
+	// Fetch the requested statues by IDs from the database.
+	statuses, err := p.state.DB.GetStatusesByIDs(ctx, statusIDs)
+	if err != nil {
+		err := gtserror.Newf("db error getting status(es): %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// Check for empty return.
+	if len(statuses) == 0 {
+		return nil, nil
+	}
+
+	// Enqueue refresh of all statuses.
+	for _, status := range statuses {
+		p.federator.Dereferencer.RefreshStatusAsync(ctx,
+			requester.Username, status, nil, nil)
+	}
+
+	// Perform visibility checks and return appropriate API models.
+	return p.c.GetVisibleAPIStatuses(ctx, requester, statuses, 0), nil
+}
+
 // SourceGet returns the *apimodel.StatusSource version of the targetStatusID.
 // Status must belong to the requester, and must not be a boost.
 func (p *Processor) SourceGet(ctx context.Context, requester *gtsmodel.Account, statusID string) (*apimodel.StatusSource, gtserror.WithCode) {

internal/processing/workers/fromclientapi_test.go

@@ -2109,6 +2109,7 @@ func (suite *FromClientAPITestSuite) TestProcessUpdateStatusInteractedWith() {
 	// it would be for real.
 	testStatus.EditIDs = []string{edit.ID}
 	testStatus.Edits = []*gtsmodel.StatusEdit{edit}
+	testStatus.Edited = true
 
 	// Update the status.
 	if err := testStructs.Processor.Workers().ProcessFromClientAPI(
@@ -2127,8 +2128,7 @@ func (suite *FromClientAPITestSuite) TestProcessUpdateStatusInteractedWith() {
 	var notif *gtsmodel.Notification
 	if !testrig.WaitFor(func() bool {
 		var err error
-		notif, err = testStructs.State.DB.GetNotification(
-			ctx,
+		notif, err = testStructs.State.DB.GetNotification(ctx,
 			gtsmodel.NotificationUpdate,
 			receivingAccount.ID,
 			postingAccount.ID,

internal/processing/workers/fromfediapi.go

@@ -239,6 +239,7 @@ func (p *Processor) ProcessFromFediAPI(ctx context.Context, fMsg *messages.FromF
 // It is also capable of handling impolite reply requests to local + remote statuses,
 // ie., replies sent directly without doing the ReplyRequest process first.
 func (p *fediAPI) CreateStatus(ctx context.Context, fMsg *messages.FromFediAPI) error {
+
 	// If we received this status via our instance account,
 	// it must originate from a relay actor we subscribe to.
 	if fMsg.Receiving.IsInstance() {
@@ -273,17 +274,16 @@ func (p *fediAPI) CreateStatus(ctx context.Context, fMsg *messages.FromFediAPI)
 		// Call RefreshStatus() to parse and process the provided
 		// statusable model, which it will use to further flesh out
 		// the bare bones model and insert it into the database.
+		//
+		// NOTE: dereferencer hook will handle surfacing logic.
 		status, statusable, err = p.federate.RefreshStatus(ctx,
 			fMsg.Receiving.Username,
 			bareStatus,
 			statusable,
+
 			// Force refresh
 			// within 5min window.
-			dereferencing.Fresh,
-			// Pass callback to insert
-			// other statuses in thread
-			// into timelines (as appropriate).
-			p.surfacer.TimelineAndNotifyStatus,
+			&dereferencing.Fresh,
 		)
 		if err != nil {
 			return gtserror.Newf("error processing new status %s: %w", bareStatus.URI, err)
@@ -292,13 +292,11 @@ func (p *fediAPI) CreateStatus(ctx context.Context, fMsg *messages.FromFediAPI)
 	case fMsg.APIRI != nil:
 		// Model was not set, deref with IRI (this is a forward).
 		// This will also cause the status to be inserted into the db.
-		status, statusable, _, err = p.federate.GetStatusByURI(ctx,
+		//
+		// NOTE: dereferencer hook will handle surfacing logic.
+		status, statusable, err = p.federate.GetStatusByURI(ctx,
 			fMsg.Receiving.Username,
 			fMsg.APIRI,
-			// Pass callback to insert
-			// other statuses in thread
-			// into timelines (as appropriate).
-			p.surfacer.TimelineAndNotifyStatus,
 		)
 		if err != nil {
 			return gtserror.Newf("error dereferencing forwarded status %s: %w", fMsg.APIRI, err)
@@ -393,12 +391,11 @@ func (p *fediAPI) CreateStatus(ctx context.Context, fMsg *messages.FromFediAPI)
 			return gtserror.Newf("error federating pre-approval of reply: %w", err)
 		}
 
-		// Don't return, just continue
-		// side effects as normal.
-	}
-
-	if err := p.surfacer.TimelineAndNotifyStatus(ctx, status); err != nil {
-		log.Errorf(ctx, "error timelining and notifying status: %v", err)
+		// Timeline and notify the status, since the missing ApprovedByURI during
+		// status dereferencer stage will cause it to skip typical surfacing logic.
+		if err := p.surfacer.TimelineAndNotifyStatus(ctx, status); err != nil {
+			log.Errorf(ctx, "error timelining and notifying status: %v", err)
+		}
 	}
 
 	return nil
@@ -411,10 +408,6 @@ func (p *fediAPI) createStatusFromRelay(ctx context.Context, fMsg *messages.From
 		fMsg.Receiving,  // Our instance account.
 		fMsg.Requesting, // Relaying account.
 		fMsg.APIRI,      // Relayed status ID/URI.
-		// Pass callback to insert
-		// other statuses in thread
-		// into timelines (as appropriate).
-		p.surfacer.TimelineAndNotifyStatus,
 	)
 
 	uriStr := fMsg.APIRI.String()
@@ -451,12 +444,6 @@ func (p *fediAPI) createStatusFromRelay(ctx context.Context, fMsg *messages.From
 		return nil
 	}
 
-	// Status was in the db already or was
-	// inserted into the db, do side effects.
-	if err := p.surfacer.TimelineAndNotifyStatus(ctx, status); err != nil {
-		l.Errorf("error timelining and notifying status: %v", err)
-	}
-
 	return nil
 }
 
@@ -482,12 +469,7 @@ func (p *fediAPI) CreateReplyRequest(ctx context.Context, fMsg *messages.FromFed
 
 		// Force refresh
 		// within 5min window.
-		dereferencing.Fresh,
-
-		// Don't pass callback;
-		// we're only interested
-		// in enriching the reply.
-		nil,
+		&dereferencing.Fresh,
 	)
 
 	switch {
@@ -570,7 +552,8 @@ func (p *fediAPI) CreateReplyRequest(ctx context.Context, fMsg *messages.FromFed
 		log.Errorf(ctx, "error federating accept: %v", err)
 	}
 
-	// Timeline the reply + notify recipient(s).
+	// Timeline and notify the status, since the missing ApprovedByURI during
+	// status dereferencer stage will cause it to skip typical surfacing logic.
 	if err := p.surfacer.TimelineAndNotifyStatus(ctx, reply); err != nil {
 		log.Errorf(ctx, "error timelining and notifying status: %v", err)
 	}
@@ -988,18 +971,9 @@ func (p *fediAPI) CreateAnnounce(ctx context.Context, fMsg *messages.FromFediAPI
 	// Note: this will handle storing the boost in
 	// the db, and dereferencing the target status
 	// ancestors / descendants where appropriate.
-	var (
-		targetIsNew bool
-		err         error
-	)
-	boost, targetIsNew, err = p.federate.EnrichAnnounce(
-		ctx,
+	boost, err := p.federate.EnrichAnnounce(ctx,
 		boost,
 		fMsg.Receiving.Username,
-		// Pass callback to insert
-		// other statuses in thread
-		// into timelines (as appropriate).
-		p.surfacer.TimelineAndNotifyStatus,
 	)
 	if err != nil {
 		if gtserror.IsUnretrievable(err) ||
@@ -1090,33 +1064,13 @@ func (p *fediAPI) CreateAnnounce(ctx context.Context, fMsg *messages.FromFediAPI
 			return gtserror.Newf("error federating pre-approval of boost: %w", err)
 		}
 
-		// Don't return, just continue
-		// side effects as normal.
-	}
-
-	// Timeline the target of the announce (if appropriate).
-	//
-	// This is done to avoid cases where we follow both the announcer
-	// of a status and the original creator of that status, and we
-	// receive the Announce of a status *before* we receive the Create
-	// of that status (say because the creator has a big queue of
-	// followers to deliver to, and someone gets it before us and
-	// boosts it to us), and we end up not timelining the original
-	// status, or notifying it, etc.
-	if targetIsNew {
-		if err := p.surfacer.TimelineAndNotifyStatus(ctx, boost.BoostOf); err != nil {
-			log.Errorf(ctx, "error timelining and notifying boosted status: %v", err)
+		// Timeline and notify the status, since the missing ApprovedByURI during
+		// status dereferencer stage will cause it to skip typical surfacing logic.
+		if err := p.surfacer.TimelineAndNotifyStatus(ctx, boost); err != nil {
+			log.Errorf(ctx, "error timelining and notifying status: %v", err)
 		}
 	}
 
-	// Timeline and notify the announce itself.
-	//
-	// This is specifically done *after* timelining the original
-	// status, so that boost depth can be taken into account.
-	if err := p.surfacer.TimelineAndNotifyStatus(ctx, boost); err != nil {
-		log.Errorf(ctx, "error timelining and notifying boost: %v", err)
-	}
-
 	if err := p.surfacer.NotifyAnnounce(ctx, boost); err != nil {
 		log.Errorf(ctx, "error notifying announce: %v", err)
 	}
@@ -1140,10 +1094,6 @@ func (p *fediAPI) createAnnounceFromRelay(ctx context.Context, fMsg *messages.Fr
 		fMsg.Receiving,  // Our instance account.
 		fMsg.Requesting, // Relaying account.
 		boost,           // Boost wrapper status.
-		// Pass callback to insert
-		// other statuses in thread
-		// into timelines (as appropriate).
-		p.surfacer.TimelineAndNotifyStatus,
 	)
 
 	uriStr := boost.BoostOfURIStr
@@ -1180,12 +1130,6 @@ func (p *fediAPI) createAnnounceFromRelay(ctx context.Context, fMsg *messages.Fr
 		return nil
 	}
 
-	// Status was in the db already or was
-	// inserted into the db, do side effects.
-	if err := p.surfacer.TimelineAndNotifyStatus(ctx, status); err != nil {
-		l.Errorf("error timelining and notifying status: %v", err)
-	}
-
 	return nil
 }
 
@@ -1200,14 +1144,9 @@ func (p *fediAPI) CreateAnnounceRequest(ctx context.Context, fMsg *messages.From
 	//
 	// We can check permissions for the announce *and*
 	// put it in the db (if acceptable) by doing Enrich.
-	boost, targetIsNew, err := p.federate.EnrichAnnounce(
-		ctx,
+	boost, err := p.federate.EnrichAnnounce(ctx,
 		req.Announce,
 		fMsg.Receiving.Username,
-		// Don't pass callback;
-		// we're only interested
-		// in enriching the announce.
-		nil,
 	)
 
 	switch {
@@ -1225,7 +1164,7 @@ func (p *fediAPI) CreateAnnounceRequest(ctx context.Context, fMsg *messages.From
 		return nil
 
 	default:
-		// There's some real error.
+		// There's a real error.
 		return gtserror.Newf(
 			"error processing AnnounceRequest with instrument %s: %w",
 			req.Announce.URI, err,
@@ -1283,24 +1222,10 @@ func (p *fediAPI) CreateAnnounceRequest(ctx context.Context, fMsg *messages.From
 		log.Errorf(ctx, "error federating accept: %v", err)
 	}
 
-	// Timeline the target of the announce (if appropriate).
-	//
-	// This is done to avoid cases where we follow both the announcer
-	// of a status and the original creator of that status, and we
-	// receive the Announce of a status *before* we receive the Create
-	// of that status (say because the creator has a big queue of
-	// followers to deliver to, and someone gets it before us and
-	// boosts it to us), and we end up not timelining the original
-	// status, or notifying it, etc.
-	if targetIsNew {
-		if err := p.surfacer.TimelineAndNotifyStatus(ctx, boost.BoostOf); err != nil {
-			log.Errorf(ctx, "error timelining and notifying boosted status: %v", err)
-		}
-	}
-
-	// Timeline the boost + notify recipient(s).
+	// Timeline and notify the boost, since the missing ApprovedByURI during
+	// status dereferencer stage will cause it to skip typical surfacing logic.
 	if err := p.surfacer.TimelineAndNotifyStatus(ctx, boost); err != nil {
-		log.Errorf(ctx, "error timelining and notifying boost: %v", err)
+		log.Errorf(ctx, "error timelining and notifying status: %v", err)
 	}
 
 	return nil
@@ -1398,12 +1323,12 @@ func (p *fediAPI) UpdateAccount(ctx context.Context, fMsg *messages.FromFediAPI)
 		account,
 		apubAcc,
 
-		// Force refresh within 5s window.
+		// Force refresh.
 		//
 		// Missing account updates could be
 		// detrimental to federation if they
 		// include public key changes.
-		dereferencing.Freshest,
+		&dereferencing.Freshest,
 	)
 	if err != nil {
 		log.Errorf(ctx, "error refreshing account: %v", err)
@@ -1467,25 +1392,16 @@ func (p *fediAPI) AcceptRemoteStatus(ctx context.Context, fMsg *messages.FromFed
 	//
 	// This will also check whether the given approvedByURI
 	// actually grants permission for this status.
-	reply, _, err := p.federate.RefreshStatus(ctx,
+	_, _, err := p.federate.RefreshStatus(ctx,
 		fMsg.Receiving.Username,
 		bareStatus,
-		nil, nil,
-		// Pass callback to insert
-		// other statuses in thread
-		// into timelines (as appropriate).
-		p.surfacer.TimelineAndNotifyStatus,
+		nil,
+		nil,
 	)
 	if err != nil {
 		return gtserror.Newf("error processing accepted status %s: %w", bareStatus.URI, err)
 	}
 
-	// No error means it was indeed a remote status, and the
-	// given approvedByURI permitted it. Timeline and notify it.
-	if err := p.surfacer.TimelineAndNotifyStatus(ctx, reply); err != nil {
-		log.Errorf(ctx, "error timelining and notifying status: %v", err)
-	}
-
 	return nil
 }
 
@@ -1565,30 +1481,21 @@ func (p *fediAPI) UpdateStatus(ctx context.Context, fMsg *messages.FromFediAPI)
 		// If an AP object was provided, we
 		// allow very fast refreshes that likely
 		// indicate a status edit after post.
-		freshness = dereferencing.Freshest
+		freshness = &dereferencing.Freshest
 	}
 
-	// Fetch up-to-date attach status attachments, etc.
-	status, _, err := p.federate.RefreshStatus(
-		ctx,
+	// Fetch up-to-date attach status attachments.
+	// NOTE: dereferencer hook handles surfacing.
+	_, _, err := p.federate.RefreshStatus(ctx,
 		fMsg.Receiving.Username,
 		existing,
 		apStatus,
 		freshness,
-		// Pass callback to insert
-		// other statuses in thread
-		// into timelines (as appropriate).
-		p.surfacer.TimelineAndNotifyStatus,
 	)
 	if err != nil {
 		log.Errorf(ctx, "error refreshing status: %v", err)
 	}
 
-	// Stream and notify relevant local users that the status has been edited.
-	if err := p.surfacer.TimelineAndNotifyStatusUpdate(ctx, status); err != nil {
-		log.Errorf(ctx, "error streaming status edit: %v", err)
-	}
-
 	return nil
 }
 

internal/processing/workers/fromfediapi_move.go

@@ -342,7 +342,7 @@ func (p *fediAPI) MoveAccount(ctx context.Context, fMsg *messages.FromFediAPI) e
 			fMsg.Receiving.Username,
 			targetAcct,
 			targetAcctable,
-			dereferencing.Freshest,
+			&dereferencing.Freshest,
 		)
 		if err != nil {
 			return gtserror.Newf(

internal/surfacing/surfacer.go

@@ -18,11 +18,15 @@
 package surfacing
 
 import (
+	"context"
+
 	"code.superseriousbusiness.org/gotosocial/internal/email"
 	"code.superseriousbusiness.org/gotosocial/internal/federation"
 	"code.superseriousbusiness.org/gotosocial/internal/filter/mutes"
 	"code.superseriousbusiness.org/gotosocial/internal/filter/status"
 	"code.superseriousbusiness.org/gotosocial/internal/filter/visibility"
+	"code.superseriousbusiness.org/gotosocial/internal/gtserror"
+	"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
 	"code.superseriousbusiness.org/gotosocial/internal/processing/conversations"
 	"code.superseriousbusiness.org/gotosocial/internal/processing/stream"
 	"code.superseriousbusiness.org/gotosocial/internal/state"
@@ -63,7 +67,7 @@ func New(
 	webPushSender webpush.Sender,
 	conversations *conversations.Processor,
 ) *Surfacer {
-	return &Surfacer{
+	s := &Surfacer{
 		state:         state,
 		converter:     converter,
 		federator:     federator,
@@ -75,4 +79,47 @@ func New(
 		webPushSender: webPushSender,
 		conversations: conversations,
 	}
+
+	// Status status dereferencer hook using surfacer.
+	federator.Dereferencer.OnStatusDereference = func(ctx context.Context, status *gtsmodel.Status, isNew bool) error {
+		if status.Flags.PendingApproval() {
+			// Status hasn't yet been
+			// approved, it needs further
+			// processing elsewhere.
+			return nil
+		}
+
+		if isNew {
+			return s.TimelineAndNotifyStatus(ctx, status)
+		} else { //nolint
+			return s.TimelineAndNotifyStatusUpdate(ctx, status)
+		}
+	}
+
+	// Set media dereferencer hook using surfacer.
+	federator.Dereferencer.OnMediaDereference = func(ctx context.Context, media *gtsmodel.MediaAttachment) error {
+		if media.StatusID == "" {
+			// we only handle this
+			// for statuses for now.
+			return nil
+		}
+
+		// Get the original status model that media is attached to.
+		status, err := state.DB.GetStatusByID(ctx, media.StatusID)
+		if err != nil {
+			return gtserror.Newf("db error getting status: %w", err)
+		}
+
+		if status.Flags.PendingApproval() {
+			// Status hasn't yet been
+			// approved, it needs further
+			// processing elsewhere.
+			return nil
+		}
+
+		// Stream a status update event with updated media.
+		return s.TimelineAndNotifyStatusUpdate(ctx, status)
+	}
+
+	return s
 }

internal/surfacing/surfacetimeline.go

@@ -169,43 +169,45 @@ func (s *Surfacer) TimelineAndNotifyStatusUpdate(ctx context.Context, status *gt
 	// event. ONLY set if an edit was received AND we
 	// successfully populated them from the database.
 	var notifyAccount func(*gtsmodel.Account)
+	if status.Edited {
 
-	// Ensure edits are fully populated for this status before anything.
-	if err := s.state.DB.PopulateStatusEdits(ctx, status); err != nil {
+		// Ensure edits are fully populated for this status before anything.
+		if err := s.state.DB.PopulateStatusEdits(ctx, status); err != nil {
 
-		// we can still continue from here, just without
-		// notifying local followers for it below here.
-		log.Error(ctx, "error populating updated status edits: %v")
+			// we can still continue from here, just without
+			// notifying local followers for it below here.
+			log.Error(ctx, "error populating updated status edits: %v")
 
-	} else if len(status.Edits) > 0 {
-		// Track accounts we've already notified this
-		// status for, as we can notify for both those
-		// having interacted with status, AND those
-		// that follow account with 'notify' flag set.
-		notified := make(map[string]struct{})
+		} else if len(status.Edits) > 0 {
+			// Track accounts we've already notified this
+			// status for, as we can notify for both those
+			// having interacted with status, AND those
+			// that follow account with 'notify' flag set.
+			notified := make(map[string]struct{})
 
-		// Don't ever notify the status author.
-		notified[status.AccountID] = struct{}{}
+			// Don't ever notify the status author.
+			notified[status.AccountID] = struct{}{}
 
-		// Get latest edit and notify for passed account.
-		latestEdit := status.Edits[len(status.Edits)-1]
-		notifyAccount = func(account *gtsmodel.Account) {
-			if _, ok := notified[account.ID]; ok {
-				return
-			}
+			// Get latest edit and notify for passed account.
+			latestEdit := status.Edits[len(status.Edits)-1]
+			notifyAccount = func(account *gtsmodel.Account) {
+				if _, ok := notified[account.ID]; ok {
+					return
+				}
 
-			// Mark account has already notified.
-			notified[account.ID] = struct{}{}
+				// Mark account has already notified.
+				notified[account.ID] = struct{}{}
 
-			// Send notif for account.
-			if err := s.Notify(ctx,
-				gtsmodel.NotificationUpdate,
-				account,
-				status.Account,
-				status,
-				latestEdit,
-			); err != nil {
-				log.Errorf(ctx, "error notifying edit for account %s: %v", account.URI, err)
+				// Send notif for account.
+				if err := s.Notify(ctx,
+					gtsmodel.NotificationUpdate,
+					account,
+					status.Account,
+					status,
+					latestEdit,
+				); err != nil {
+					log.Errorf(ctx, "error notifying edit for account %s: %v", account.URI, err)
+				}
 			}
 		}
 	}
@@ -253,6 +255,13 @@ func (s *Surfacer) TimelineAndNotifyStatusUpdate(ctx context.Context, status *gt
 		notifyAccount,
 	)
 
+	// If the status wasn't edited,
+	// it was just refreshed, we have
+	// nothing more to be done here.
+	if !status.Edited {
+		return nil
+	}
+
 	// Notify any *new* mentions added by editor.
 	for _, mention := range status.Mentions {
 

internal/typeutils/astointernal.go

@@ -450,12 +450,14 @@ func (c *Converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab
 	// with an authorization URI.
 	if status.InReplyToURI != "" {
 		var approvedByURI *url.URL
-		// Try replyAuthorization property first.
+
+		// Try extract the replyAuthorization property first.
 		if wrp, ok := statusable.(ap.WithReplyAuthorization); ok {
 			approvedByURI = ap.GetReplyAuthorization(wrp)
 		}
 
-		// Fall back to deprecated approvedBy property.
+		// Fall back to deprecated
+		// approvedBy property.
 		if approvedByURI == nil {
 			if wab, ok := statusable.(ap.WithApprovedBy); ok {
 				approvedByURI = ap.GetApprovedBy(wab)

internal/typeutils/internaltoas.go

@@ -589,7 +589,7 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
 		tagProp.AppendTootHashtag(asHashtag)
 	}
 
-	// Append built `tag` property.
+	// Append built `tag` property
 	if tagProp.Len() != 0 {
 		statusable.SetActivityStreamsTag(tagProp)
 	}