Skip to content

Commit 3cc2993

Browse files
achannarasappaachannarasappa
committed
feat: session refresh
1 parent 332527d commit 3cc2993

File tree

6 files changed

+457
-27
lines changed

6 files changed

+457
-27
lines changed

‎internal/monitor/monitor.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ func NewMonitor(configMonitor ConfigMonitor) (*Monitor, error) {
6464
monitorYahoo.Config{
6565
Ctx: ctx,
6666
UnaryURL: "https://query1.finance.yahoo.com",
67+
SessionRootURL: "https://finance.yahoo.com",
68+
SessionCrumbURL: "https://query2.finance.yahoo.com",
69+
SessionConsentURL: "https://consent.yahoo.com",
6770
ChanError: chanError,
6871
ChanUpdateAssetQuote: chanUpdateAssetQuote,
6972
},

‎internal/monitor/yahoo/monitor.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ type input struct {
4242
type Config struct {
4343
Ctx context.Context
4444
UnaryURL string
45+
SessionRootURL string
46+
SessionCrumbURL string
47+
SessionConsentURL string
4548
ChanError chan error
4649
ChanUpdateAssetQuote chan c.MessageUpdate[c.AssetQuote]
4750
}
@@ -53,7 +56,12 @@ func NewMonitorYahoo(config Config, opts ...Option) *MonitorYahoo {
5356

5457
ctx, cancel := context.WithCancel(config.Ctx)
5558

56-
unaryAPI := unary.NewUnaryAPI(config.UnaryURL)
59+
unaryAPI := unary.NewUnaryAPI(unary.Config{
60+
BaseURL: config.UnaryURL,
61+
SessionRootURL: config.SessionRootURL,
62+
SessionCrumbURL: config.SessionCrumbURL,
63+
SessionConsentURL: config.SessionConsentURL,
64+
})
5765

5866
monitor := &MonitorYahoo{
5967
assetQuotesCacheLookup: make(map[string]*c.AssetQuote),

internal/monitor/yahoo/unary/helpers.go renamed to internal/monitor/yahoo/unary/helpers-quote.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ import (
66
c "github.com/achannarasappa/ticker/v4/internal/common"
77
)
88

9+
//nolint:gochecknoglobals
10+
var (
11+
postMarketStatuses = map[string]bool{"POST": true, "POSTPOST": true}
12+
)
13+
914
// transformResponseQuote transforms a single quote returned by the API into an AssetQuote
1015
func transformResponseQuote(responseQuote ResponseQuote) c.AssetQuote {
1116

‎internal/monitor/yahoo/unary/helpers-session.go‎

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
package unary
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
"net/url"
9+
"regexp"
10+
"strings"
11+
)
12+
13+
// Constants for URLs and common header values
14+
const (
15+
sessionCrumbPath = "/v1/test/getcrumb"
16+
sessionConsentPathPattern = "/v2/collectConsent?sessionId=%s"
17+
18+
// Common header values
19+
defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"
20+
defaultAcceptValue = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
21+
defaultAcceptLang = "en-US,en;q=0.9"
22+
)
23+
24+
// refreshSession refreshes the Yahoo Finance session by getting new cookies and crumb
25+
func (u *UnaryAPI) refreshSession() error {
26+
var err error
27+
28+
// Get cookies
29+
u.cookies, err = u.getCookie()
30+
if err != nil {
31+
return err
32+
}
33+
34+
// Get crumb
35+
u.crumb, err = u.getCrumb()
36+
if err != nil {
37+
return err
38+
}
39+
40+
return nil
41+
}
42+
43+
// getCookie retrieves authentication cookies from Yahoo Finance
44+
func (u *UnaryAPI) getCookie() ([]*http.Cookie, error) {
45+
req, err := http.NewRequest("GET", u.sessionRootURL, nil)
46+
if err != nil {
47+
return nil, fmt.Errorf("error creating cookie request: %w", err)
48+
}
49+
50+
req.Header.Set("authority", "finance.yahoo.com")
51+
req.Header.Set("accept", defaultAcceptValue)
52+
req.Header.Set("accept-language", defaultAcceptLang)
53+
req.Header.Set("user-agent", defaultUserAgent)
54+
55+
resp, err := u.client.Do(req)
56+
if err != nil {
57+
return nil, fmt.Errorf("error requesting a cookie: %w", err)
58+
}
59+
defer resp.Body.Close()
60+
61+
// Check for EU consent redirect
62+
if isEUConsentRedirect(resp) {
63+
return u.getCookieEU()
64+
}
65+
66+
cookies := resp.Cookies()
67+
if !isRequiredCookieSet(cookies) {
68+
return nil, errors.New("session refresh error: A3 session cookie missing from response")
69+
}
70+
71+
return cookies, nil
72+
}
73+
74+
// getCookieEU handles the EU consent flow to get cookies
75+
func (u *UnaryAPI) getCookieEU() ([]*http.Cookie, error) {
76+
var cookies []*http.Cookie
77+
78+
client1 := u.createClientWithRedirectLimit(3)
79+
80+
// First request to get redirected to consent page
81+
req1, err := http.NewRequest("GET", u.sessionRootURL, nil)
82+
if err != nil {
83+
return nil, fmt.Errorf("error creating EU consent request: %w", err)
84+
}
85+
86+
req1.Header.Set("authority", "finance.yahoo.com")
87+
req1.Header.Set("accept", defaultAcceptValue)
88+
req1.Header.Set("accept-language", defaultAcceptLang)
89+
req1.Header.Set("user-agent", defaultUserAgent)
90+
91+
resp1, err := client1.Do(req1)
92+
if err != nil {
93+
return nil, fmt.Errorf("error attempting to get Yahoo API session id: %w", err)
94+
}
95+
defer resp1.Body.Close()
96+
97+
if resp1.StatusCode < 200 || resp1.StatusCode >= 300 {
98+
return nil, fmt.Errorf("session refresh error: unexpected response from Yahoo API: non-2xx response code: %d", resp1.StatusCode)
99+
}
100+
101+
// Extract session ID and CSRF token from URL
102+
sessionID, csrfToken, err := extractSessionAndCSRF(resp1)
103+
if err != nil {
104+
return nil, err
105+
}
106+
107+
// Get GUCS cookie
108+
gucsCookies := resp1.Cookies()
109+
if len(gucsCookies) == 0 {
110+
return nil, errors.New("session refresh error: no cookies set by finance.yahoo.com")
111+
}
112+
113+
// Submit consent form
114+
formData := url.Values{}
115+
formData.Set("csrfToken", csrfToken)
116+
formData.Set("sessionId", sessionID)
117+
formData.Set("namespace", "yahoo")
118+
formData.Set("agree", "agree")
119+
120+
formDataStr := formData.Encode()
121+
122+
req2, err := http.NewRequest("POST", u.sessionConsentURL+fmt.Sprintf(sessionConsentPathPattern, sessionID), strings.NewReader(formDataStr))
123+
if err != nil {
124+
return nil, fmt.Errorf("error creating consent submission request: %w", err)
125+
}
126+
127+
req2.Header.Set("origin", "https://consent.yahoo.com")
128+
req2.Header.Set("host", "consent.yahoo.com")
129+
req2.Header.Set("content-type", "application/x-www-form-urlencoded")
130+
req2.Header.Set("accept", defaultAcceptValue)
131+
req2.Header.Set("accept-language", defaultAcceptLang)
132+
req2.Header.Set("referer", u.sessionConsentURL+fmt.Sprintf(sessionConsentPathPattern, sessionID))
133+
req2.Header.Set("user-agent", defaultUserAgent)
134+
req2.Header.Set("content-length", fmt.Sprintf("%d", len(formDataStr)))
135+
req2.Header.Set("accept-encoding", "gzip, deflate, br")
136+
req2.Header.Set("dnt", "1")
137+
req2.Header.Set("sec-ch-ua", "\"Google Chrome\";v=\"134\", \"Chromium\";v=\"134\", \"Not-A.Brand\";v=\"24\"")
138+
req2.Header.Set("sec-ch-ua-mobile", "?0")
139+
req2.Header.Set("sec-ch-ua-platform", "\"Windows\"")
140+
req2.Header.Set("sec-fetch-dest", "document")
141+
req2.Header.Set("sec-fetch-mode", "navigate")
142+
req2.Header.Set("sec-fetch-site", "same-origin")
143+
req2.Header.Set("sec-fetch-user", "?1")
144+
145+
// Add GUCS cookies
146+
for _, cookie := range gucsCookies {
147+
req2.AddCookie(cookie)
148+
}
149+
150+
client2 := u.createClientWithRedirectLimit(2)
151+
152+
resp2, err := client2.Do(req2)
153+
if err != nil {
154+
return nil, fmt.Errorf("error attempting to agree to EU consent request: %w", err)
155+
}
156+
defer resp2.Body.Close()
157+
158+
cookies = resp2.Cookies()
159+
if !isRequiredCookieSet(cookies) {
160+
return nil, errors.New("session refresh error: A3 session cookie missing from response after agreeing to EU consent request")
161+
}
162+
163+
return cookies, nil
164+
}
165+
166+
// getCrumb retrieves the crumb value needed for authenticated requests
167+
func (u *UnaryAPI) getCrumb() (string, error) {
168+
req, err := http.NewRequest("GET", u.sessionCrumbURL+sessionCrumbPath, nil)
169+
if err != nil {
170+
return "", fmt.Errorf("error creating crumb request: %w", err)
171+
}
172+
173+
req.Header.Set("authority", "query2.finance.yahoo.com")
174+
req.Header.Set("accept", "*/*")
175+
req.Header.Set("accept-language", defaultAcceptLang)
176+
req.Header.Set("content-type", "text/plain")
177+
req.Header.Set("origin", u.sessionRootURL)
178+
req.Header.Set("user-agent", defaultUserAgent)
179+
180+
// Add cookies
181+
for _, cookie := range u.cookies {
182+
req.AddCookie(cookie)
183+
}
184+
185+
resp, err := u.client.Do(req)
186+
if err != nil {
187+
return "", fmt.Errorf("error requesting a crumb: %w", err)
188+
}
189+
defer resp.Body.Close()
190+
191+
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
192+
return "", fmt.Errorf("session refresh error: unexpected response from Yahoo API when attempting to retrieve crumb: non-2xx response code: %d", resp.StatusCode)
193+
}
194+
195+
// Read crumb from response body
196+
crumbBytes, err := io.ReadAll(resp.Body)
197+
if err != nil {
198+
return "", fmt.Errorf("error reading crumb response: %w", err)
199+
}
200+
201+
return string(crumbBytes), nil
202+
}
203+
204+
// createClientWithRedirectLimit returns a new http.Client with the specified redirect limit
205+
func (a *UnaryAPI) createClientWithRedirectLimit(limit int) *http.Client {
206+
return &http.Client{
207+
CheckRedirect: func(req *http.Request, via []*http.Request) error {
208+
if len(via) >= limit {
209+
return http.ErrUseLastResponse
210+
}
211+
return nil
212+
},
213+
}
214+
}
215+
216+
// isEUConsentRedirect checks if the response is a redirect to the EU consent page
217+
func isEUConsentRedirect(resp *http.Response) bool {
218+
return strings.Contains(resp.Header.Get("Location"), "guce.yahoo.com") &&
219+
resp.StatusCode >= 300 && resp.StatusCode < 400
220+
}
221+
222+
// isRequiredCookieSet checks if the A3 cookie is present in the cookies
223+
func isRequiredCookieSet(cookies []*http.Cookie) bool {
224+
for _, cookie := range cookies {
225+
if cookie.Name == "A3" {
226+
return true
227+
}
228+
}
229+
return false
230+
}
231+
232+
// extractSessionAndCSRF extracts session ID and CSRF token from response
233+
func extractSessionAndCSRF(resp *http.Response) (string, string, error) {
234+
// Extract session ID from URL
235+
sessionIDRegex := regexp.MustCompile("sessionId=(?:([A-Za-z0-9_-]*))")
236+
csrfTokenRegex := regexp.MustCompile("gcrumb=(?:([A-Za-z0-9_]*))")
237+
238+
// Check for session ID in Location header or URL
239+
var sessionIDMatch []string
240+
var csrfTokenMatch []string
241+
242+
if resp.Request.URL != nil {
243+
sessionIDMatch = sessionIDRegex.FindStringSubmatch(resp.Request.URL.String())
244+
}
245+
246+
if len(sessionIDMatch) < 2 {
247+
return "", "", errors.New("session refresh error: error unable to extract session id from redirected request URL")
248+
}
249+
250+
// Check for CSRF token in URL
251+
if resp.Request.Response != nil && resp.Request.Response.Request != nil && resp.Request.Response.Request.URL != nil {
252+
csrfTokenMatch = csrfTokenRegex.FindStringSubmatch(resp.Request.Response.Request.URL.String())
253+
}
254+
255+
if len(csrfTokenMatch) < 2 {
256+
return "", "", errors.New("session refresh error: error unable to extract CSRF token from Location header")
257+
}
258+
259+
return sessionIDMatch[1], csrfTokenMatch[1], nil
260+
}

0 commit comments

Comments
 (0)