Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/fetcher/company_names.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ func cleanCompanyName(name string) string {
n = strings.ReplaceAll(n, ",", "")
for _, suffix := range []string{
" inc", " inc.", " corp", " corp.", " corporation", " co", " co.",
" ltd", " ltd.", " limited", " plc", " ag", " sa", " se", " holdings", " group",
" ltd", " ltd.", " limited", " llc", " llc.", " plc", " ag", " sa", " se", " holdings", " group",
} {
if strings.HasSuffix(n, suffix) {
n = strings.TrimSpace(strings.TrimSuffix(n, suffix))
Expand Down
266 changes: 246 additions & 20 deletions internal/fetcher/job_fetcher.go

Large diffs are not rendered by default.

18 changes: 18 additions & 0 deletions internal/fetcher/job_fetcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2338,3 +2338,21 @@ func TestFetchAllJobsCombinesLLMAndRSSSources(t *testing.T) {
t.Fatalf("jobs[1].Company = %q; want Unknown", jobs[1].Company)
}
}

func TestJobMatchesCompanyTargetUsesWebsiteAndAliases(t *testing.T) {
scope := companyFetchScope{
Company: "GitHub",
Aliases: []string{"GitHub Inc"},
Website: "https://github.com",
}

if !jobMatchesCompanyTarget(Job{Company: "GitHub Inc"}, scope) {
t.Fatal("jobMatchesCompanyTarget() rejected alias company without website")
}
if !jobMatchesCompanyTarget(Job{Company: "Other", CompanyWebsite: "https://github.com/about"}, scope) {
t.Fatal("jobMatchesCompanyTarget() rejected matching provided website")
}
if jobMatchesCompanyTarget(Job{Company: "GitHub", CompanyWebsite: "https://gitlab.com"}, scope) {
t.Fatal("jobMatchesCompanyTarget() accepted conflicting discovered website")
}
}
75 changes: 75 additions & 0 deletions internal/fetcher/llm_web_search.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,46 @@ func buildLLMWebSearchPrompt(criteria *CriteriaConfig, targets []string) (string
return b.String(), queries
}

func buildCompanyLLMWebSearchPrompt(criteria *CriteriaConfig, targets []string, company companyFetchScope, matchCriteria bool) (string, []string) {
queries := llmWebSearchQueriesForCompany(criteria, targets, company, matchCriteria, maxLLMWebSearchQueries)
if len(queries) == 0 {
return "", nil
}

var b strings.Builder
b.WriteString("Use provider-backed web search, if available, to find current public job postings.\n")
b.WriteString("If web search is not available in this runtime, return exactly [] with no explanation.\n")
b.WriteString("Do not answer with prose, caveats, markdown, or citations outside the JSON array.\n")
b.WriteString("When a provider web search tool is available, use it before deciding there are no matching jobs.\n")
fmt.Fprintf(&b, "Only include roles where the employer is %s.\n", strings.TrimSpace(company.Company))
if len(company.Aliases) > 0 {
fmt.Fprintf(&b, "Also treat these names as the same employer: %s.\n", strings.Join(company.Aliases, ", "))
}
if domain := company.websiteDomain(); domain != "" {
fmt.Fprintf(&b, "The company's official website domain is %s. Exclude roles whose source evidence points to a different company website.\n", domain)
}
b.WriteString("Search only these public-web queries:\n")
for _, query := range queries {
fmt.Fprintf(&b, "- %s\n", query)
}
if domains := llmWebSearchDomains(targets); len(domains) > 0 {
b.WriteString("\nAllowed source domains for providers that support domain filters:\n")
for _, domain := range domains {
fmt.Fprintf(&b, "- %s\n", domain)
}
}
if matchCriteria {
b.WriteString("\nOnly include roles that match the criteria below. Prefer direct employer or ATS application pages. Exclude stale, closed, unrelated, senior/lead/manager roles when those are excluded by criteria.\n")
b.WriteString("For each job, include the actual company website, a brief factual company summary, and the company industry when available.\n\n")
writeLLMWebCriteria(&b, criteria)
} else {
b.WriteString("\nReturn all current public roles for this company. Do not filter by candidate criteria.\n")
b.WriteString("Prefer direct employer or ATS application pages. For each job, include the actual company website, a brief factual company summary, and the company industry when available.\n")
}

return b.String(), queries
}

func llmWebSearchQueries(criteria *CriteriaConfig, targets []string, limit int) []string {
titleQueries := targetedSiteSearchQueries(criteria)
if len(titleQueries) == 0 {
Expand Down Expand Up @@ -83,6 +123,41 @@ func llmWebSearchQueries(criteria *CriteriaConfig, targets []string, limit int)
return queries
}

func llmWebSearchQueriesForCompany(criteria *CriteriaConfig, targets []string, company companyFetchScope, matchCriteria bool, limit int) []string {
searches := companyTargetSearchQueries(company, criteria, matchCriteria)
if len(searches) == 0 {
return nil
}

queries := make([]string, 0)
seen := make(map[string]bool)
normalizedTargets := make([]string, 0, len(targets))
for _, rawTarget := range targets {
if target := llmWebSearchTarget(rawTarget); target != "" {
normalizedTargets = append(normalizedTargets, target)
}
}

for _, search := range searches {
for _, target := range normalizedTargets {
query := strings.TrimSpace(target + " " + strings.TrimSpace(search))
if query == "" {
continue
}
key := strings.ToLower(query)
if seen[key] {
continue
}
seen[key] = true
queries = append(queries, query)
if limit > 0 && len(queries) >= limit {
return queries
}
}
}
return queries
}

func llmWebSearchDomains(targets []string) []string {
domains := make([]string, 0, len(targets))
seen := make(map[string]bool)
Expand Down
26 changes: 26 additions & 0 deletions internal/fetcher/llm_web_search_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,32 @@ func TestBuildLLMWebSearchPromptIncludesProviderWebInstruction(t *testing.T) {
}
}

func TestBuildCompanyLLMWebSearchPromptTargetsCompany(t *testing.T) {
criteria := &CriteriaConfig{}
criteria.Filters.TitleIncludes = []string{"Software Engineer"}

scope := companyFetchScope{Company: "GitHub", Website: "https://github.com"}
prompt, queries := buildCompanyLLMWebSearchPrompt(criteria, []string{"site:jobs.ashbyhq.com"}, scope, true)
wantQueries := []string{"site:jobs.ashbyhq.com GitHub github.com Software Engineer"}
if strings.Join(queries, "\x00") != strings.Join(wantQueries, "\x00") {
t.Fatalf("buildCompanyLLMWebSearchPrompt(match criteria) queries = %#v; want %#v", queries, wantQueries)
}
for _, want := range []string{"Only include roles where the employer is GitHub", "github.com", "GitHub github.com Software Engineer"} {
if !strings.Contains(prompt, want) {
t.Fatalf("buildCompanyLLMWebSearchPrompt(match criteria) prompt missing %q in:\n%s", want, prompt)
}
}

prompt, queries = buildCompanyLLMWebSearchPrompt(criteria, []string{"site:jobs.ashbyhq.com"}, scope, false)
wantQueries = []string{"site:jobs.ashbyhq.com GitHub github.com"}
if strings.Join(queries, "\x00") != strings.Join(wantQueries, "\x00") {
t.Fatalf("buildCompanyLLMWebSearchPrompt(all) queries = %#v; want %#v", queries, wantQueries)
}
if strings.Contains(prompt, "Target titles: Software Engineer") {
t.Fatalf("buildCompanyLLMWebSearchPrompt(all) prompt includes criteria title:\n%s", prompt)
}
}

func TestLLMWebSearchDomainsNormalizesTargets(t *testing.T) {
got := llmWebSearchDomains([]string{
"site:job-boards.greenhouse.io",
Expand Down
165 changes: 165 additions & 0 deletions internal/fetcher/site_search_browser.go
Original file line number Diff line number Diff line change
Expand Up @@ -912,6 +912,108 @@
return []string{u.String()}
}

func siteSearchURLsForCompany(target string, criteria *CriteriaConfig, company companyFetchScope, matchCriteria bool) []string {
targetURL := siteSearchURL(target)
if targetURL == "" {
return nil
}
u, err := url.Parse(targetURL)
if err != nil {
return []string{targetURL}
}

searches := companyTargetSearchQueries(company, criteria, matchCriteria)
if len(searches) == 0 {
return nil
}
searchCriteria := criteria
if !matchCriteria {
searchCriteria = nil
}
searchKey := ""
switch {
case isBuiltInHost(u.Hostname()):
if u.Path == "" || u.Path == "/" {
u.Path = "/jobs"
}
query := u.Query()
if query.Get("search") == "" {
searchKey = "search"
}
if query.Get("country") == "" {
if country := builtInCountryParam(searchCriteria); country != "" {
query.Set("country", country)
}
}
if searchCriteria != nil && searchCriteria.Filters.WorkSettings.Remote && isBuiltInNationalHost(u.Hostname()) && query.Get("allLocations") == "" {
query.Set("allLocations", "true")
}
u.RawQuery = query.Encode()
case isIndeedHost(u.Hostname()):
if u.Path == "" || u.Path == "/" {
u.Path = "/jobs"
}
query := u.Query()
if query.Get("q") == "" {
searchKey = "q"
}
if query.Get("l") == "" {
if location := siteSearchLocation(searchCriteria); location != "" {
query.Set("l", location)
}
}
u.RawQuery = query.Encode()
case isLinkedInHost(u.Hostname()):
if u.Path == "" || u.Path == "/" || u.Path == "/jobs" {
u.Path = "/jobs/search"
}
query := u.Query()
if query.Get("keywords") == "" {
searchKey = "keywords"
}
if query.Get("location") == "" {
if location := linkedInLocationQuery(searchCriteria); location != "" {
query.Set("location", location)
}
}
if query.Get("f_PP") == "" {
if geoID := linkedInGeoID(searchCriteria); geoID != "" {
query.Set("f_PP", geoID)
}
}
if query.Get("f_WT") == "" {
if workplaceTypes := linkedInWorkplaceTypes(searchCriteria); workplaceTypes != "" {
query.Set("f_WT", workplaceTypes)
}
}
if query.Get("f_E") == "" {
if experienceLevels := linkedInExperienceLevels(searchCriteria); experienceLevels != "" {
query.Set("f_E", experienceLevels)
}
}
if query.Get("f_SB2") == "" {
if salaryBucket := linkedInSalaryBucket(searchCriteria); salaryBucket != "" {
query.Set("f_SB2", salaryBucket)
}
}
u.RawQuery = query.Encode()
default:
return []string{u.String()}
}
var urls []string
if searchKey != "" {
urls = siteSearchURLsWithQueryValues(u, searchKey, searches)
} else {
urls = []string{u.String()}
}
if isIndeedHost(u.Hostname()) {
for _, companyURL := range indeedCompanySearchURLs(company) {
urls = appendUniqueString(urls, companyURL)
}
}
return urls
}

func siteSearchURLsWithQueryValues(base *url.URL, key string, values []string) []string {
if base == nil {
return nil
Expand Down Expand Up @@ -962,6 +1064,69 @@
return queries
}

func companyTargetSearchQueries(company companyFetchScope, criteria *CriteriaConfig, matchCriteria bool) []string {
names := company.names()
if len(names) == 0 {
return nil
}
domain := company.websiteDomain()
baseQueries := make([]string, 0, len(names))
for _, name := range names {
query := strings.TrimSpace(name)
if domain != "" {
query = strings.TrimSpace(query + " " + domain)
}
baseQueries = appendUniqueString(baseQueries, query)
}
if !matchCriteria {
return baseQueries
}
titleQueries := targetedSiteSearchQueries(criteria)
if len(titleQueries) == 0 {
return baseQueries
}
queries := make([]string, 0, len(baseQueries)*len(titleQueries))

Check failure

Code scanning / CodeQL

Size computation for allocation may overflow High

This operation, which is used in an
allocation
, involves a
potentially large value
and might overflow.
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
seen := make(map[string]bool)
for _, baseQuery := range baseQueries {
for _, titleQuery := range titleQueries {
query := strings.TrimSpace(baseQuery + " " + strings.TrimSpace(titleQuery))
if query == "" {
continue
}
key := strings.ToLower(query)
if seen[key] {
continue
}
seen[key] = true
queries = append(queries, query)
}
}
if len(queries) == 0 {
return baseQueries
}
return queries
}

func indeedCompanySearchURLs(company companyFetchScope) []string {
names := company.names()
if len(names) == 0 {
return nil
}
urls := make([]string, 0, len(names))
for _, name := range names {
u := url.URL{
Scheme: "https",
Host: "www.indeed.com",
Path: "/companies/search",
}
query := u.Query()
query.Set("q", name)
u.RawQuery = query.Encode()
urls = appendUniqueString(urls, u.String())
}
return urls
}

func combinedTitleSearchQuery(prefix string, title string) string {
prefix = strings.TrimSpace(prefix)
title = strings.TrimSpace(title)
Expand Down
31 changes: 31 additions & 0 deletions internal/fetcher/site_search_browser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ func TestSiteSearchURLForCriteriaBuildsAggregatorSearchURLs(t *testing.T) {
criteria.Filters.TitleIncludes = []string{"Software Engineer"}
criteria.Filters.WorkSettings.Remote = true
criteria.Filters.WorkSettings.Hybrid = true
criteria.Filters.WorkSettings.Hybrid = true
Comment thread
Copilot marked this conversation as resolved.
Outdated
criteria.Candidate.YearsOfExperience = 5
criteria.Filters.MinBaseUSD = 120000

Expand Down Expand Up @@ -219,6 +220,36 @@ func TestSiteSearchURLForCriteriaBuildsAggregatorSearchURLs(t *testing.T) {
}
}

func TestSiteSearchURLsForCompanyBuildsCompanyQueries(t *testing.T) {
criteria := &CriteriaConfig{}
criteria.Candidate.City = "Seattle"
criteria.Candidate.State = "WA"
criteria.Candidate.CountryCode = "US"
criteria.Filters.TitleRequires = []string{"Senior"}
criteria.Filters.TitleIncludes = []string{"Software Engineer"}
criteria.Filters.WorkSettings.Remote = true
criteria.Filters.WorkSettings.Hybrid = true

scope := companyFetchScope{Company: "GitHub", Website: "https://github.com"}
got := siteSearchURLsForCompany("https://www.indeed.com/jobs", criteria, scope, true)
want := []string{
"https://www.indeed.com/jobs?l=Seattle%2C+WA&q=GitHub+github.com+Senior+Software+Engineer",
"https://www.indeed.com/companies/search?q=GitHub",
}
if strings.Join(got, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("siteSearchURLsForCompany(match criteria) = %#v; want %#v", got, want)
}

got = siteSearchURLsForCompany("https://www.indeed.com/jobs", criteria, scope, false)
want = []string{
"https://www.indeed.com/jobs?q=GitHub+github.com",
"https://www.indeed.com/companies/search?q=GitHub",
}
if strings.Join(got, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("siteSearchURLsForCompany(all) = %#v; want %#v", got, want)
}
}

func TestSiteSearchURLsForCriteriaBuildsTargetedTitleMatrix(t *testing.T) {
criteria := &CriteriaConfig{}
criteria.Candidate.City = "Seattle"
Expand Down
Loading