diff --git a/harnesses/aggregator-head-lag/cmd/scrape-cookie/main.go b/harnesses/aggregator-head-lag/cmd/scrape-cookie/main.go new file mode 100644 index 00000000..686fa5d6 --- /dev/null +++ b/harnesses/aggregator-head-lag/cmd/scrape-cookie/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" +) + +func main() { + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + ) + + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + defer cancel() + ctx, cancel := chromedp.NewContext(allocCtx) + defer cancel() + ctx, cancel = context.WithTimeout(ctx, 45*time.Second) + defer cancel() + + cookies := map[string]string{} + + err := chromedp.Run(ctx, + chromedp.Navigate("https://www.defined.fi/"), + chromedp.WaitVisible(`body`, chromedp.ByQuery), + chromedp.Sleep(8*time.Second), + chromedp.ActionFunc(func(ctx context.Context) error { + cookieParams, err := network.GetCookies().Do(ctx) + if err != nil { + return fmt.Errorf("failed to get cookies: %w", err) + } + for _, cookie := range cookieParams { + cookies[cookie.Name] = cookie.Value + fmt.Fprintf(os.Stderr, "cookie: %s (len=%d)\n", cookie.Name, len(cookie.Value)) + } + return nil + }), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "chrome error: %v\n", err) + os.Exit(1) + } + + // Try codex_token first (it's a pre-minted token, might be usable directly) + // It's stored as URL-encoded JSON: {"token":""} + if raw, ok := cookies["codex_token"]; ok { + decoded, err := url.QueryUnescape(raw) + if err == nil { + var obj struct { + Token string `json:"token"` + } + if json.Unmarshal([]byte(decoded), &obj) == nil && obj.Token != "" { + fmt.Fprintf(os.Stderr, "\ncodex_token JWT (len=%d): %s...\n", len(obj.Token), obj.Token[:min(60, len(obj.Token))]) + // Test if we can use this token directly for a Codex query + testCodexToken(obj.Token) + } + } + } + + // Also test defined-attestation-token for JWT minting + if attToken, ok := cookies["defined-attestation-token"]; ok { + fmt.Fprintf(os.Stderr, "\nTesting defined-attestation-token for JWT mint...\n") + testJWTMint(attToken, "defined-attestation-token") + } + + // Output fresh defined-attestation-token + if attToken, ok := cookies["defined-attestation-token"]; ok { + fmt.Printf("DEFINED_SESSION_COOKIE=%s\n", attToken) + fmt.Printf("COOKIE_NAME=defined-attestation-token\n") + } + + // Also output full codex_token JSON decoded token + if raw, ok := cookies["codex_token"]; ok { + decoded, _ := url.QueryUnescape(raw) + var obj struct { + Token string `json:"token"` + } + if json.Unmarshal([]byte(decoded), &obj) == nil { + fmt.Printf("CODEX_TOKEN=%s\n", obj.Token) + } + } +} + +func testJWTMint(cookieValue, cookieName string) { + reqBody := map[string]interface{}{ + "operationName": "CreateApiToken", + "query": "mutation CreateApiToken { createApiTokens(input: { count: 1 }) { token } }", + "variables": map[string]interface{}{}, + } + bodyBytes, _ := json.Marshal(reqBody) + req, _ := http.NewRequest("POST", "https://www.defined.fi/api", bytes.NewBuffer(bodyBytes)) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Origin", "https://www.defined.fi") + req.Header.Set("Referer", "https://www.defined.fi/") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req.AddCookie(&http.Cookie{Name: cookieName, Value: cookieValue}) + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + fmt.Fprintf(os.Stderr, "mint request failed: %v\n", err) + return + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + fmt.Fprintf(os.Stderr, "mint status: %d, body: %s\n", resp.StatusCode, string(body[:min(200, len(body))])) +} + +func testCodexToken(token string) { + // Try using the token directly as a Bearer for Codex GraphQL + reqBody := map[string]interface{}{ + "query": "{ getNetworkStats(networkId: 1) { addressCount } }", + } + bodyBytes, _ := json.Marshal(reqBody) + req, _ := http.NewRequest("POST", "https://graph.codex.io/graphql", bytes.NewBuffer(bodyBytes)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + fmt.Fprintf(os.Stderr, "codex token test failed: %v\n", err) + return + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + fmt.Fprintf(os.Stderr, "codex test status: %d, body: %s\n", resp.StatusCode, string(body[:min(200, len(body))])) +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/harnesses/aggregator-head-lag/cmd/script/defined_auth.go b/harnesses/aggregator-head-lag/cmd/script/defined_auth.go index 716f05f0..717e75ad 100644 --- a/harnesses/aggregator-head-lag/cmd/script/defined_auth.go +++ b/harnesses/aggregator-head-lag/cmd/script/defined_auth.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "sync" "time" @@ -57,8 +58,12 @@ func decodeJWTExpiration(token string) (time.Time, error) { return time.Unix(claims.Exp, 0), nil } -// GetDefinedJWTToken returns a cached JWT token or generates a new one if expired +// GetDefinedJWTToken returns a cached JWT token or generates a new one if expired. +// If CODEX_JWT env var is set, it is returned directly (bypasses session-cookie flow). func GetDefinedJWTToken(sessionCookie string) (string, error) { + if jwt := os.Getenv("CODEX_JWT"); jwt != "" { + return jwt, nil + } globalTokenCache.mu.RLock() // Check if we have a valid cached token @@ -144,7 +149,7 @@ func generateDefinedJWTToken(sessionCookie string) (string, error) { req.Header.Set("sec-fetch-dest", "empty") req.Header.Set("sec-fetch-mode", "cors") req.Header.Set("sec-fetch-site", "same-origin") - req.AddCookie(&http.Cookie{Name: "session", Value: sessionCookie}) + req.AddCookie(&http.Cookie{Name: "defined-attestation-token", Value: sessionCookie}) fmt.Println("[DEFINED-AUTH] Sending POST request to https://www.defined.fi/api...") resp, err := client.Do(req) diff --git a/harnesses/aggregator-head-lag/cmd/script/scrape_session.go b/harnesses/aggregator-head-lag/cmd/script/scrape_session.go index 4607912c..d88746e6 100644 --- a/harnesses/aggregator-head-lag/cmd/script/scrape_session.go +++ b/harnesses/aggregator-head-lag/cmd/script/scrape_session.go @@ -42,13 +42,13 @@ func ScrapeDefinedSessionCookie() (string, error) { } for _, cookie := range cookieParams { - if cookie.Name == "session" { + if cookie.Name == "defined-attestation-token" { sessionCookie = cookie.Value return nil } } - return fmt.Errorf("session cookie not found") + return fmt.Errorf("defined-attestation-token cookie not found") }), ) diff --git a/harnesses/aggregator-latency-benchmark/cmd/script/defined_auth.go b/harnesses/aggregator-latency-benchmark/cmd/script/defined_auth.go index 716f05f0..717e75ad 100644 --- a/harnesses/aggregator-latency-benchmark/cmd/script/defined_auth.go +++ b/harnesses/aggregator-latency-benchmark/cmd/script/defined_auth.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "sync" "time" @@ -57,8 +58,12 @@ func decodeJWTExpiration(token string) (time.Time, error) { return time.Unix(claims.Exp, 0), nil } -// GetDefinedJWTToken returns a cached JWT token or generates a new one if expired +// GetDefinedJWTToken returns a cached JWT token or generates a new one if expired. +// If CODEX_JWT env var is set, it is returned directly (bypasses session-cookie flow). func GetDefinedJWTToken(sessionCookie string) (string, error) { + if jwt := os.Getenv("CODEX_JWT"); jwt != "" { + return jwt, nil + } globalTokenCache.mu.RLock() // Check if we have a valid cached token @@ -144,7 +149,7 @@ func generateDefinedJWTToken(sessionCookie string) (string, error) { req.Header.Set("sec-fetch-dest", "empty") req.Header.Set("sec-fetch-mode", "cors") req.Header.Set("sec-fetch-site", "same-origin") - req.AddCookie(&http.Cookie{Name: "session", Value: sessionCookie}) + req.AddCookie(&http.Cookie{Name: "defined-attestation-token", Value: sessionCookie}) fmt.Println("[DEFINED-AUTH] Sending POST request to https://www.defined.fi/api...") resp, err := client.Do(req) diff --git a/harnesses/aggregator-latency-benchmark/cmd/script/scrape_session.go b/harnesses/aggregator-latency-benchmark/cmd/script/scrape_session.go index 4607912c..26d3f770 100644 --- a/harnesses/aggregator-latency-benchmark/cmd/script/scrape_session.go +++ b/harnesses/aggregator-latency-benchmark/cmd/script/scrape_session.go @@ -42,7 +42,7 @@ func ScrapeDefinedSessionCookie() (string, error) { } for _, cookie := range cookieParams { - if cookie.Name == "session" { + if cookie.Name == "defined-attestation-token" { sessionCookie = cookie.Value return nil } diff --git a/harnesses/metadata-coverage/cmd/script/defined_auth.go b/harnesses/metadata-coverage/cmd/script/defined_auth.go index 716f05f0..717e75ad 100644 --- a/harnesses/metadata-coverage/cmd/script/defined_auth.go +++ b/harnesses/metadata-coverage/cmd/script/defined_auth.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "sync" "time" @@ -57,8 +58,12 @@ func decodeJWTExpiration(token string) (time.Time, error) { return time.Unix(claims.Exp, 0), nil } -// GetDefinedJWTToken returns a cached JWT token or generates a new one if expired +// GetDefinedJWTToken returns a cached JWT token or generates a new one if expired. +// If CODEX_JWT env var is set, it is returned directly (bypasses session-cookie flow). func GetDefinedJWTToken(sessionCookie string) (string, error) { + if jwt := os.Getenv("CODEX_JWT"); jwt != "" { + return jwt, nil + } globalTokenCache.mu.RLock() // Check if we have a valid cached token @@ -144,7 +149,7 @@ func generateDefinedJWTToken(sessionCookie string) (string, error) { req.Header.Set("sec-fetch-dest", "empty") req.Header.Set("sec-fetch-mode", "cors") req.Header.Set("sec-fetch-site", "same-origin") - req.AddCookie(&http.Cookie{Name: "session", Value: sessionCookie}) + req.AddCookie(&http.Cookie{Name: "defined-attestation-token", Value: sessionCookie}) fmt.Println("[DEFINED-AUTH] Sending POST request to https://www.defined.fi/api...") resp, err := client.Do(req) diff --git a/harnesses/metadata-coverage/cmd/script/scrape_session.go b/harnesses/metadata-coverage/cmd/script/scrape_session.go index 4607912c..26d3f770 100644 --- a/harnesses/metadata-coverage/cmd/script/scrape_session.go +++ b/harnesses/metadata-coverage/cmd/script/scrape_session.go @@ -42,7 +42,7 @@ func ScrapeDefinedSessionCookie() (string, error) { } for _, cookie := range cookieParams { - if cookie.Name == "session" { + if cookie.Name == "defined-attestation-token" { sessionCookie = cookie.Value return nil } diff --git a/harnesses/network-coverage/cmd/script/codex.go b/harnesses/network-coverage/cmd/script/codex.go index db31dde1..60db0b81 100644 --- a/harnesses/network-coverage/cmd/script/codex.go +++ b/harnesses/network-coverage/cmd/script/codex.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "os" "strconv" "time" ) @@ -99,7 +100,11 @@ func fetchCodex(cfg *Config) ProviderResult { // getCodexJWT mints a Defined.fi JWT from the session cookie, or pulls a // pre-minted one from a sidecar URL if DEFINED_TOKEN_SERVICE_URL is set. +// If CODEX_JWT env var is set, it is returned directly (bypasses all other flows). func getCodexJWT(cfg *Config) (string, error) { + if jwt := os.Getenv("CODEX_JWT"); jwt != "" { + return jwt, nil + } // If a sidecar token service is configured, try it first. On any error, // silently fall back to inline mint — the sidecar isn't authoritative. if cfg.DefinedTokenURL != "" { @@ -127,7 +132,7 @@ func getCodexJWT(cfg *Config) (string, error) { req.Header.Set("Accept", "application/json") req.Header.Set("Origin", "https://www.defined.fi") req.Header.Set("Referer", "https://www.defined.fi/") - req.Header.Set("Cookie", "session-token="+cfg.CodexSessionCookie) + req.AddCookie(&http.Cookie{Name: "defined-attestation-token", Value: cfg.CodexSessionCookie}) resp, err := client.Do(req) if err != nil { diff --git a/harnesses/pm-rate-limits/cmd/script/codex_auth.go b/harnesses/pm-rate-limits/cmd/script/codex_auth.go index b06ed1eb..a4e1fd39 100644 --- a/harnesses/pm-rate-limits/cmd/script/codex_auth.go +++ b/harnesses/pm-rate-limits/cmd/script/codex_auth.go @@ -14,6 +14,7 @@ import ( "io" "log" "net/http" + "os" "strings" "sync" "time" @@ -59,7 +60,11 @@ func decodeJWTExpiration(token string) (time.Time, error) { // GetCodexJWT returns a cached short-lived JWT, minting a new one when the // cached one is within 1h of expiry. +// If CODEX_JWT env var is set, it is returned directly (bypasses session-cookie flow). func GetCodexJWT(sessionCookie string) (string, error) { + if jwt := os.Getenv("CODEX_JWT"); jwt != "" { + return jwt, nil + } codexTokenCache.mu.RLock() if codexTokenCache.token != "" && time.Now().Before(codexTokenCache.expiresAt.Add(-1*time.Hour)) { t := codexTokenCache.token @@ -120,7 +125,7 @@ func mintCodexJWT(sessionCookie string) (string, error) { req.Header.Set("sec-fetch-dest", "empty") req.Header.Set("sec-fetch-mode", "cors") req.Header.Set("sec-fetch-site", "same-origin") - req.AddCookie(&http.Cookie{Name: "session", Value: sessionCookie}) + req.AddCookie(&http.Cookie{Name: "defined-attestation-token", Value: sessionCookie}) resp, err := client.Do(req) if err != nil {