feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
raymondjacobson merged 1 commit into from
Jun 9, 2026
Merged

feat(api): consume blocks via gRPC StreamBlocks (flagged, default off) #941

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
ArchiverNodes []string
Rewards []rewards.Reward
AudiusdURL string
CoreBlockStreamEnabled bool
OpenAudioURLs []string
ChainId string
BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
AxiomDataset: os.Getenv("axiomDataset"),
NetworkTakeRate: 10,
AudiusdURL: os.Getenv("audiusdUrl"),
CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
OpenAudioURLs: []string{},
BirdeyeToken: os.Getenv("birdeyeToken"),
EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
connectrpc.com/connect v1.18.1
github.com/AlecAivazis/survey/v2 v2.3.7
github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
github.com/aquasecurity/esquery v0.2.0
github.com/axiomhq/axiom-go v0.23.0
github.com/axiomhq/hyperloglog v0.2.5
Expand Down
12 changes: 4 additions & 8 deletions go.sum
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down
26 changes: 26 additions & 0 deletions indexer/indexer.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
"context"
"fmt"
"net/http"
"strings"
"time"

"api.audius.co/config"
dbv1 "api.audius.co/database"
"api.audius.co/jobs"
"api.audius.co/logging"
"connectrpc.com/connect"
corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
etl "github.com/OpenAudio/go-openaudio/pkg/etl"
em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
etlCfg.DisableMaterializedViewRefresh()
etlCfg.DisablePgNotifyListener()
etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

etlIndexer := etl.New(openAudioSDK.Core, logger)
etlIndexer.SetConfig(etlCfg)
etlIndexer.SetDBURL(cfg.WriteDbUrl)
etlIndexer.SetCheckReadiness(true)

// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
// instead of polling GetBlocks. The ETL falls back to polling automatically
// if the endpoint doesn't support it, so this is safe to flip on per-env.
// Kept separate from the SDK's unary Core client (used for status + fallback).
if cfg.CoreBlockStreamEnabled {
etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
}

// Restore the pre-vendor setPubkeyForUser behavior via the upstream
// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
baseURL := audiusdURL
if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
baseURL = "https://" + baseURL
}
return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raymondjacobson
merged 1 commit into

from

Jun 9, 2026
 Merged


feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)
#941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
config/config.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
	ArchiverNodes []string
	Rewards []rewards.Reward
	AudiusdURL string
	CoreBlockStreamEnabled bool
	OpenAudioURLs []string
	ChainId string
	BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
	AxiomDataset: os.Getenv("axiomDataset"),
	NetworkTakeRate: 10,
	AudiusdURL: os.Getenv("audiusdUrl"),
	CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
	OpenAudioURLs: []string{},
	BirdeyeToken: os.Getenv("birdeyeToken"),
	EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down

4 changes: 2 additions & 2 deletions
go.mod

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
	connectrpc.com/connect v1.18.1
	github.com/AlecAivazis/survey/v2 v2.3.7
	github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
	github.com/aquasecurity/esquery v0.2.0
	github.com/axiomhq/axiom-go v0.23.0
	github.com/axiomhq/hyperloglog v0.2.5
Expand Down

12 changes: 4 additions & 8 deletions
go.sum

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down

26 changes: 26 additions & 0 deletions
indexer/indexer.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"api.audius.co/config"
	dbv1 "api.audius.co/database"
	"api.audius.co/jobs"
	"api.audius.co/logging"
	"connectrpc.com/connect"
	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
	etl "github.com/OpenAudio/go-openaudio/pkg/etl"
	em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
	"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	etlCfg.DisableMaterializedViewRefresh()
	etlCfg.DisablePgNotifyListener()
	etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
	etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

	etlIndexer := etl.New(openAudioSDK.Core, logger)
	etlIndexer.SetConfig(etlCfg)
	etlIndexer.SetDBURL(cfg.WriteDbUrl)
	etlIndexer.SetCheckReadiness(true)

	// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
	// instead of polling GetBlocks. The ETL falls back to polling automatically
	// if the endpoint doesn't support it, so this is safe to flip on per-env.
	// Kept separate from the SDK's unary Core client (used for status + fallback).
	if cfg.CoreBlockStreamEnabled {
 etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
 logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
	}

	// Restore the pre-vendor setPubkeyForUser behavior via the upstream
	// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
	// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
	baseURL := audiusdURL
	if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
 baseURL = "https://" + baseURL
	}
	return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raymondjacobson
merged 1 commit into

from

Jun 9, 2026
 Merged


feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)
#941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
config/config.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
	ArchiverNodes []string
	Rewards []rewards.Reward
	AudiusdURL string
	CoreBlockStreamEnabled bool
	OpenAudioURLs []string
	ChainId string
	BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
	AxiomDataset: os.Getenv("axiomDataset"),
	NetworkTakeRate: 10,
	AudiusdURL: os.Getenv("audiusdUrl"),
	CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
	OpenAudioURLs: []string{},
	BirdeyeToken: os.Getenv("birdeyeToken"),
	EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down

4 changes: 2 additions & 2 deletions
go.mod

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
	connectrpc.com/connect v1.18.1
	github.com/AlecAivazis/survey/v2 v2.3.7
	github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
	github.com/aquasecurity/esquery v0.2.0
	github.com/axiomhq/axiom-go v0.23.0
	github.com/axiomhq/hyperloglog v0.2.5
Expand Down

12 changes: 4 additions & 8 deletions
go.sum

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down

26 changes: 26 additions & 0 deletions
indexer/indexer.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"api.audius.co/config"
	dbv1 "api.audius.co/database"
	"api.audius.co/jobs"
	"api.audius.co/logging"
	"connectrpc.com/connect"
	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
	etl "github.com/OpenAudio/go-openaudio/pkg/etl"
	em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
	"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	etlCfg.DisableMaterializedViewRefresh()
	etlCfg.DisablePgNotifyListener()
	etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
	etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

	etlIndexer := etl.New(openAudioSDK.Core, logger)
	etlIndexer.SetConfig(etlCfg)
	etlIndexer.SetDBURL(cfg.WriteDbUrl)
	etlIndexer.SetCheckReadiness(true)

	// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
	// instead of polling GetBlocks. The ETL falls back to polling automatically
	// if the endpoint doesn't support it, so this is safe to flip on per-env.
	// Kept separate from the SDK's unary Core client (used for status + fallback).
	if cfg.CoreBlockStreamEnabled {
 etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
 logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
	}

	// Restore the pre-vendor setPubkeyForUser behavior via the upstream
	// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
	// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
	baseURL := audiusdURL
	if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
 baseURL = "https://" + baseURL
	}
	return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raymondjacobson
merged 1 commit into

from

Jun 9, 2026
 Merged


feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)
#941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
config/config.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
	ArchiverNodes []string
	Rewards []rewards.Reward
	AudiusdURL string
	CoreBlockStreamEnabled bool
	OpenAudioURLs []string
	ChainId string
	BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
	AxiomDataset: os.Getenv("axiomDataset"),
	NetworkTakeRate: 10,
	AudiusdURL: os.Getenv("audiusdUrl"),
	CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
	OpenAudioURLs: []string{},
	BirdeyeToken: os.Getenv("birdeyeToken"),
	EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down

4 changes: 2 additions & 2 deletions
go.mod

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
	connectrpc.com/connect v1.18.1
	github.com/AlecAivazis/survey/v2 v2.3.7
	github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
	github.com/aquasecurity/esquery v0.2.0
	github.com/axiomhq/axiom-go v0.23.0
	github.com/axiomhq/hyperloglog v0.2.5
Expand Down

12 changes: 4 additions & 8 deletions
go.sum

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down

26 changes: 26 additions & 0 deletions
indexer/indexer.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"api.audius.co/config"
	dbv1 "api.audius.co/database"
	"api.audius.co/jobs"
	"api.audius.co/logging"
	"connectrpc.com/connect"
	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
	etl "github.com/OpenAudio/go-openaudio/pkg/etl"
	em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
	"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	etlCfg.DisableMaterializedViewRefresh()
	etlCfg.DisablePgNotifyListener()
	etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
	etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

	etlIndexer := etl.New(openAudioSDK.Core, logger)
	etlIndexer.SetConfig(etlCfg)
	etlIndexer.SetDBURL(cfg.WriteDbUrl)
	etlIndexer.SetCheckReadiness(true)

	// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
	// instead of polling GetBlocks. The ETL falls back to polling automatically
	// if the endpoint doesn't support it, so this is safe to flip on per-env.
	// Kept separate from the SDK's unary Core client (used for status + fallback).
	if cfg.CoreBlockStreamEnabled {
 etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
 logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
	}

	// Restore the pre-vendor setPubkeyForUser behavior via the upstream
	// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
	// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
	baseURL := audiusdURL
	if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
 baseURL = "https://" + baseURL
	}
	return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raymondjacobson
merged 1 commit into

from

Jun 9, 2026
 Merged


feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)
#941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
config/config.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
	ArchiverNodes []string
	Rewards []rewards.Reward
	AudiusdURL string
	CoreBlockStreamEnabled bool
	OpenAudioURLs []string
	ChainId string
	BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
	AxiomDataset: os.Getenv("axiomDataset"),
	NetworkTakeRate: 10,
	AudiusdURL: os.Getenv("audiusdUrl"),
	CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
	OpenAudioURLs: []string{},
	BirdeyeToken: os.Getenv("birdeyeToken"),
	EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down

4 changes: 2 additions & 2 deletions
go.mod

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
	connectrpc.com/connect v1.18.1
	github.com/AlecAivazis/survey/v2 v2.3.7
	github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
	github.com/aquasecurity/esquery v0.2.0
	github.com/axiomhq/axiom-go v0.23.0
	github.com/axiomhq/hyperloglog v0.2.5
Expand Down

12 changes: 4 additions & 8 deletions
go.sum

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down

26 changes: 26 additions & 0 deletions
indexer/indexer.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"api.audius.co/config"
	dbv1 "api.audius.co/database"
	"api.audius.co/jobs"
	"api.audius.co/logging"
	"connectrpc.com/connect"
	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
	etl "github.com/OpenAudio/go-openaudio/pkg/etl"
	em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
	"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	etlCfg.DisableMaterializedViewRefresh()
	etlCfg.DisablePgNotifyListener()
	etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
	etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

	etlIndexer := etl.New(openAudioSDK.Core, logger)
	etlIndexer.SetConfig(etlCfg)
	etlIndexer.SetDBURL(cfg.WriteDbUrl)
	etlIndexer.SetCheckReadiness(true)

	// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
	// instead of polling GetBlocks. The ETL falls back to polling automatically
	// if the endpoint doesn't support it, so this is safe to flip on per-env.
	// Kept separate from the SDK's unary Core client (used for status + fallback).
	if cfg.CoreBlockStreamEnabled {
 etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
 logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
	}

	// Restore the pre-vendor setPubkeyForUser behavior via the upstream
	// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
	// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
	baseURL := audiusdURL
	if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
 baseURL = "https://" + baseURL
	}
	return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raymondjacobson
merged 1 commit into

from

Jun 9, 2026
 Merged


feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)
#941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
config/config.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
	ArchiverNodes []string
	Rewards []rewards.Reward
	AudiusdURL string
	CoreBlockStreamEnabled bool
	OpenAudioURLs []string
	ChainId string
	BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
	AxiomDataset: os.Getenv("axiomDataset"),
	NetworkTakeRate: 10,
	AudiusdURL: os.Getenv("audiusdUrl"),
	CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
	OpenAudioURLs: []string{},
	BirdeyeToken: os.Getenv("birdeyeToken"),
	EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down

4 changes: 2 additions & 2 deletions
go.mod

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
	connectrpc.com/connect v1.18.1
	github.com/AlecAivazis/survey/v2 v2.3.7
	github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
	github.com/aquasecurity/esquery v0.2.0
	github.com/axiomhq/axiom-go v0.23.0
	github.com/axiomhq/hyperloglog v0.2.5
Expand Down

12 changes: 4 additions & 8 deletions
go.sum

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down

26 changes: 26 additions & 0 deletions
indexer/indexer.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"api.audius.co/config"
	dbv1 "api.audius.co/database"
	"api.audius.co/jobs"
	"api.audius.co/logging"
	"connectrpc.com/connect"
	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
	etl "github.com/OpenAudio/go-openaudio/pkg/etl"
	em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
	"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	etlCfg.DisableMaterializedViewRefresh()
	etlCfg.DisablePgNotifyListener()
	etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
	etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

	etlIndexer := etl.New(openAudioSDK.Core, logger)
	etlIndexer.SetConfig(etlCfg)
	etlIndexer.SetDBURL(cfg.WriteDbUrl)
	etlIndexer.SetCheckReadiness(true)

	// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
	// instead of polling GetBlocks. The ETL falls back to polling automatically
	// if the endpoint doesn't support it, so this is safe to flip on per-env.
	// Kept separate from the SDK's unary Core client (used for status + fallback).
	if cfg.CoreBlockStreamEnabled {
 etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
 logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
	}

	// Restore the pre-vendor setPubkeyForUser behavior via the upstream
	// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
	// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
	baseURL := audiusdURL
	if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
 baseURL = "https://" + baseURL
	}
	return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raymondjacobson
merged 1 commit into

from

Jun 9, 2026
 Merged


feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)
#941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
config/config.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
	ArchiverNodes []string
	Rewards []rewards.Reward
	AudiusdURL string
	CoreBlockStreamEnabled bool
	OpenAudioURLs []string
	ChainId string
	BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
	AxiomDataset: os.Getenv("axiomDataset"),
	NetworkTakeRate: 10,
	AudiusdURL: os.Getenv("audiusdUrl"),
	CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
	OpenAudioURLs: []string{},
	BirdeyeToken: os.Getenv("birdeyeToken"),
	EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down

4 changes: 2 additions & 2 deletions
go.mod

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
	connectrpc.com/connect v1.18.1
	github.com/AlecAivazis/survey/v2 v2.3.7
	github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
	github.com/aquasecurity/esquery v0.2.0
	github.com/axiomhq/axiom-go v0.23.0
	github.com/axiomhq/hyperloglog v0.2.5
Expand Down

12 changes: 4 additions & 8 deletions
go.sum

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down

26 changes: 26 additions & 0 deletions
indexer/indexer.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"api.audius.co/config"
	dbv1 "api.audius.co/database"
	"api.audius.co/jobs"
	"api.audius.co/logging"
	"connectrpc.com/connect"
	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
	etl "github.com/OpenAudio/go-openaudio/pkg/etl"
	em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
	"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	etlCfg.DisableMaterializedViewRefresh()
	etlCfg.DisablePgNotifyListener()
	etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
	etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

	etlIndexer := etl.New(openAudioSDK.Core, logger)
	etlIndexer.SetConfig(etlCfg)
	etlIndexer.SetDBURL(cfg.WriteDbUrl)
	etlIndexer.SetCheckReadiness(true)

	// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
	// instead of polling GetBlocks. The ETL falls back to polling automatically
	// if the endpoint doesn't support it, so this is safe to flip on per-env.
	// Kept separate from the SDK's unary Core client (used for status + fallback).
	if cfg.CoreBlockStreamEnabled {
 etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
 logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
	}

	// Restore the pre-vendor setPubkeyForUser behavior via the upstream
	// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
	// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
	baseURL := audiusdURL
	if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
 baseURL = "https://" + baseURL
	}
	return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)#941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raymondjacobson
merged 1 commit into

from

Jun 9, 2026
 Merged


feat(api): consume blocks via gRPC StreamBlocks (flagged, default off)
#941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
config/config.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ type Config struct {
	ArchiverNodes []string
	Rewards []rewards.Reward
	AudiusdURL string
	CoreBlockStreamEnabled bool
	OpenAudioURLs []string
	ChainId string
	BirdeyeToken string
Expand DownExpand Up@@ -95,6 +96,7 @@ var Cfg = Config{
	AxiomDataset: os.Getenv("axiomDataset"),
	NetworkTakeRate: 10,
	AudiusdURL: os.Getenv("audiusdUrl"),
	CoreBlockStreamEnabled: os.Getenv("coreBlockStreamEnabled") == "true",
	OpenAudioURLs: []string{},
	BirdeyeToken: os.Getenv("birdeyeToken"),
	EthRpcUrl: os.Getenv("ethRpcUrl"),
Expand Down

4 changes: 2 additions & 2 deletions
go.mod

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,8 +6,8 @@ require (
	connectrpc.com/connect v1.18.1
	github.com/AlecAivazis/survey/v2 v2.3.7
	github.com/Doist/unfurlist v0.0.0-20250409100812-515f2735f8e5
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6
	github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b
	github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b
	github.com/aquasecurity/esquery v0.2.0
	github.com/axiomhq/axiom-go v0.23.0
	github.com/axiomhq/hyperloglog v0.2.5
Expand Down

12 changes: 4 additions & 8 deletions
go.sum

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,14 +20,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5 h1:ncJjkgXNaL7r8LyNGigHONOgFVq+jBpkMKtgyZrju1o=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6 h1:lRaOXeltp59DNPe9+btz7HeqxOl5Edf0AK/ePT9khak=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5 h1:l4KjBR+wR5DpHPV03N0DKFLcPsMhP3Hq82JquBL5Ajo=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609040102-bda0b8b592f5/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6 h1:vuZ8SeGb2tQemF5478KsPl1gPqNif8airkZSaehud2I=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609193221-97ccec32c1c6/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b h1:vlK/jCRaB8E/0pKgVVanP7hSrESH4JSYl/+dZfwYQoA=
github.com/OpenAudio/go-openaudio v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:wiFXmVbIUkN2D5lRshknaARCKhzbHtCBKRCZe6UOnVs=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b h1:z77PTyEzh/CT6n31U5fQ1Pn7AzttGaduzqvBk8xgAqM=
github.com/OpenAudio/go-openaudio/pkg/etl v1.3.1-0.20260609211151-aee578ec923b/go.mod h1:LZKiU9vBYzlZzn6oPRHHLPXteBtMKQPegNH9bX9JuH8=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
Expand Down

26 changes: 26 additions & 0 deletions
indexer/indexer.go

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@ package indexer
import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"api.audius.co/config"
	dbv1 "api.audius.co/database"
	"api.audius.co/jobs"
	"api.audius.co/logging"
	"connectrpc.com/connect"
	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
	etl "github.com/OpenAudio/go-openaudio/pkg/etl"
	em "github.com/OpenAudio/go-openaudio/pkg/etl/processors/entity_manager"
	"github.com/OpenAudio/go-openaudio/pkg/sdk"
Expand DownExpand Up@@ -68,12 +72,22 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	etlCfg.DisableMaterializedViewRefresh()
	etlCfg.DisablePgNotifyListener()
	etlCfg.ReadDataTypesEnv() // honors OPENAUDIO_ETL_ENTITY_MANAGER_DATA_TYPES if set
	etlCfg.BlockStreamEnabled = cfg.CoreBlockStreamEnabled

	etlIndexer := etl.New(openAudioSDK.Core, logger)
	etlIndexer.SetConfig(etlCfg)
	etlIndexer.SetDBURL(cfg.WriteDbUrl)
	etlIndexer.SetCheckReadiness(true)

	// When enabled, source blocks from the CoreService.StreamBlocks gRPC stream
	// instead of polling GetBlocks. The ETL falls back to polling automatically
	// if the endpoint doesn't support it, so this is safe to flip on per-env.
	// Kept separate from the SDK's unary Core client (used for status + fallback).
	if cfg.CoreBlockStreamEnabled {
 etlIndexer.SetBlockStreamClient(newCoreStreamClient(cfg.AudiusdURL))
 logger.Info("etl block source: gRPC stream", zap.String("endpoint", cfg.AudiusdURL))
	}

	// Restore the pre-vendor setPubkeyForUser behavior via the upstream
	// post-create hook (go-openaudio #317). Recovers the EIP-712 pubkey
	// from each User Create tx and writes it to user_pubkeys in the same
Expand DownExpand Up@@ -108,6 +122,18 @@ func NewIndexer(cfg config.Config) *CoreIndexer {
	}
}

// newCoreStreamClient builds a gRPC (HTTP/2) CoreService client for the block
// stream. connect.WithGRPC requires HTTP/2, which the default transport
// negotiates over TLS. This is separate from the SDK's unary Core client, which
// the ETL still uses for status checks and the polling fallback.
func newCoreStreamClient(audiusdURL string) corev1connect.CoreServiceClient {
	baseURL := audiusdURL
	if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
 baseURL = "https://" + baseURL
	}
	return corev1connect.NewCoreServiceClient(http.DefaultClient, baseURL, connect.WithGRPC())
}

// Start runs the ETL indexer alongside the aggregates calculator. Both are
// long-lived; errgroup propagates the first error (and the ctx cancellation
// it triggers) to all members.
Expand Down
Loading