Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/fetcher/company_names.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ func cleanCompanyName(name string) string {
n = strings.ReplaceAll(n, ",", "")
for _, suffix := range []string{
" inc", " inc.", " corp", " corp.", " corporation", " co", " co.",
" ltd", " ltd.", " limited", " plc", " ag", " sa", " se", " holdings", " group",
" ltd", " ltd.", " limited", " llc", " llc.", " plc", " ag", " sa", " se", " holdings", " group",
} {
if strings.HasSuffix(n, suffix) {
n = strings.TrimSpace(strings.TrimSuffix(n, suffix))
Expand Down
266 changes: 246 additions & 20 deletions internal/fetcher/job_fetcher.go

Large diffs are not rendered by default.

18 changes: 18 additions & 0 deletions internal/fetcher/job_fetcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2338,3 +2338,21 @@ func TestFetchAllJobsCombinesLLMAndRSSSources(t *testing.T) {
t.Fatalf("jobs[1].Company = %q; want Unknown", jobs[1].Company)
}
}

func TestJobMatchesCompanyTargetUsesWebsiteAndAliases(t *testing.T) {
scope := companyFetchScope{
Company: "GitHub",
Aliases: []string{"GitHub Inc"},
Website: "https://github.com",
}

if !jobMatchesCompanyTarget(Job{Company: "GitHub Inc"}, scope) {
t.Fatal("jobMatchesCompanyTarget() rejected alias company without website")
}
if !jobMatchesCompanyTarget(Job{Company: "Other", CompanyWebsite: "https://github.com/about"}, scope) {
t.Fatal("jobMatchesCompanyTarget() rejected matching provided website")
}
if jobMatchesCompanyTarget(Job{Company: "GitHub", CompanyWebsite: "https://gitlab.com"}, scope) {
t.Fatal("jobMatchesCompanyTarget() accepted conflicting discovered website")
}
}
75 changes: 75 additions & 0 deletions internal/fetcher/llm_web_search.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,46 @@ func buildLLMWebSearchPrompt(criteria *CriteriaConfig, targets []string) (string
return b.String(), queries
}

func buildCompanyLLMWebSearchPrompt(criteria *CriteriaConfig, targets []string, company companyFetchScope, matchCriteria bool) (string, []string) {
queries := llmWebSearchQueriesForCompany(criteria, targets, company, matchCriteria, maxLLMWebSearchQueries)
if len(queries) == 0 {
return "", nil
}

var b strings.Builder
b.WriteString("Use provider-backed web search, if available, to find current public job postings.\n")
b.WriteString("If web search is not available in this runtime, return exactly [] with no explanation.\n")
b.WriteString("Do not answer with prose, caveats, markdown, or citations outside the JSON array.\n")
b.WriteString("When a provider web search tool is available, use it before deciding there are no matching jobs.\n")
fmt.Fprintf(&b, "Only include roles where the employer is %s.\n", strings.TrimSpace(company.Company))
if len(company.Aliases) > 0 {
fmt.Fprintf(&b, "Also treat these names as the same employer: %s.\n", strings.Join(company.Aliases, ", "))
}
if domain := company.websiteDomain(); domain != "" {
fmt.Fprintf(&b, "The company's official website domain is %s. Exclude roles whose source evidence points to a different company website.\n", domain)
}
b.WriteString("Search only these public-web queries:\n")
for _, query := range queries {
fmt.Fprintf(&b, "- %s\n", query)
}
if domains := llmWebSearchDomains(targets); len(domains) > 0 {
b.WriteString("\nAllowed source domains for providers that support domain filters:\n")
for _, domain := range domains {
fmt.Fprintf(&b, "- %s\n", domain)
}
}
if matchCriteria {
b.WriteString("\nOnly include roles that match the criteria below. Prefer direct employer or ATS application pages. Exclude stale, closed, unrelated, senior/lead/manager roles when those are excluded by criteria.\n")
b.WriteString("For each job, include the actual company website, a brief factual company summary, and the company industry when available.\n\n")
writeLLMWebCriteria(&b, criteria)
} else {
b.WriteString("\nReturn all current public roles for this company. Do not filter by candidate criteria.\n")
b.WriteString("Prefer direct employer or ATS application pages. For each job, include the actual company website, a brief factual company summary, and the company industry when available.\n")
}

return b.String(), queries
}

func llmWebSearchQueries(criteria *CriteriaConfig, targets []string, limit int) []string {
titleQueries := targetedSiteSearchQueries(criteria)
if len(titleQueries) == 0 {
Expand Down Expand Up @@ -83,6 +123,41 @@ func llmWebSearchQueries(criteria *CriteriaConfig, targets []string, limit int)
return queries
}

func llmWebSearchQueriesForCompany(criteria *CriteriaConfig, targets []string, company companyFetchScope, matchCriteria bool, limit int) []string {
searches := companyTargetSearchQueries(company, criteria, matchCriteria)
if len(searches) == 0 {
return nil
}

queries := make([]string, 0)
seen := make(map[string]bool)
normalizedTargets := make([]string, 0, len(targets))
for _, rawTarget := range targets {
if target := llmWebSearchTarget(rawTarget); target != "" {
normalizedTargets = append(normalizedTargets, target)
}
}

for _, search := range searches {
for _, target := range normalizedTargets {
query := strings.TrimSpace(target + " " + strings.TrimSpace(search))
if query == "" {
continue
}
key := strings.ToLower(query)
if seen[key] {
continue
}
seen[key] = true
queries = append(queries, query)
if limit > 0 && len(queries) >= limit {
return queries
}
}
}
return queries
}

func llmWebSearchDomains(targets []string) []string {
domains := make([]string, 0, len(targets))
seen := make(map[string]bool)
Expand Down
26 changes: 26 additions & 0 deletions internal/fetcher/llm_web_search_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,32 @@ func TestBuildLLMWebSearchPromptIncludesProviderWebInstruction(t *testing.T) {
}
}

func TestBuildCompanyLLMWebSearchPromptTargetsCompany(t *testing.T) {
criteria := &CriteriaConfig{}
criteria.Filters.TitleIncludes = []string{"Software Engineer"}

scope := companyFetchScope{Company: "GitHub", Website: "https://github.com"}
prompt, queries := buildCompanyLLMWebSearchPrompt(criteria, []string{"site:jobs.ashbyhq.com"}, scope, true)
wantQueries := []string{"site:jobs.ashbyhq.com GitHub github.com Software Engineer"}
if strings.Join(queries, "\x00") != strings.Join(wantQueries, "\x00") {
t.Fatalf("buildCompanyLLMWebSearchPrompt(match criteria) queries = %#v; want %#v", queries, wantQueries)
}
for _, want := range []string{"Only include roles where the employer is GitHub", "github.com", "GitHub github.com Software Engineer"} {
if !strings.Contains(prompt, want) {
t.Fatalf("buildCompanyLLMWebSearchPrompt(match criteria) prompt missing %q in:\n%s", want, prompt)
}
}

prompt, queries = buildCompanyLLMWebSearchPrompt(criteria, []string{"site:jobs.ashbyhq.com"}, scope, false)
wantQueries = []string{"site:jobs.ashbyhq.com GitHub github.com"}
if strings.Join(queries, "\x00") != strings.Join(wantQueries, "\x00") {
t.Fatalf("buildCompanyLLMWebSearchPrompt(all) queries = %#v; want %#v", queries, wantQueries)
}
if strings.Contains(prompt, "Target titles: Software Engineer") {
t.Fatalf("buildCompanyLLMWebSearchPrompt(all) prompt includes criteria title:\n%s", prompt)
}
}

func TestLLMWebSearchDomainsNormalizesTargets(t *testing.T) {
got := llmWebSearchDomains([]string{
"site:job-boards.greenhouse.io",
Expand Down
165 changes: 165 additions & 0 deletions internal/fetcher/site_search_browser.go
Original file line number Diff line number Diff line change
Expand Up @@ -912,6 +912,108 @@ func siteSearchURLsForCriteria(target string, criteria *CriteriaConfig) []string
return []string{u.String()}
}

func siteSearchURLsForCompany(target string, criteria *CriteriaConfig, company companyFetchScope, matchCriteria bool) []string {
targetURL := siteSearchURL(target)
if targetURL == "" {
return nil
}
u, err := url.Parse(targetURL)
if err != nil {
return []string{targetURL}
}

searches := companyTargetSearchQueries(company, criteria, matchCriteria)
if len(searches) == 0 {
return nil
}
searchCriteria := criteria
if !matchCriteria {
searchCriteria = nil
}
searchKey := ""
switch {
case isBuiltInHost(u.Hostname()):
if u.Path == "" || u.Path == "/" {
u.Path = "/jobs"
}
query := u.Query()
if query.Get("search") == "" {
searchKey = "search"
}
if query.Get("country") == "" {
if country := builtInCountryParam(searchCriteria); country != "" {
query.Set("country", country)
}
}
if searchCriteria != nil && searchCriteria.Filters.WorkSettings.Remote && isBuiltInNationalHost(u.Hostname()) && query.Get("allLocations") == "" {
query.Set("allLocations", "true")
}
u.RawQuery = query.Encode()
case isIndeedHost(u.Hostname()):
if u.Path == "" || u.Path == "/" {
u.Path = "/jobs"
}
query := u.Query()
if query.Get("q") == "" {
searchKey = "q"
}
if query.Get("l") == "" {
if location := siteSearchLocation(searchCriteria); location != "" {
query.Set("l", location)
}
}
u.RawQuery = query.Encode()
case isLinkedInHost(u.Hostname()):
if u.Path == "" || u.Path == "/" || u.Path == "/jobs" {
u.Path = "/jobs/search"
}
query := u.Query()
if query.Get("keywords") == "" {
searchKey = "keywords"
}
if query.Get("location") == "" {
if location := linkedInLocationQuery(searchCriteria); location != "" {
query.Set("location", location)
}
}
if query.Get("f_PP") == "" {
if geoID := linkedInGeoID(searchCriteria); geoID != "" {
query.Set("f_PP", geoID)
}
}
if query.Get("f_WT") == "" {
if workplaceTypes := linkedInWorkplaceTypes(searchCriteria); workplaceTypes != "" {
query.Set("f_WT", workplaceTypes)
}
}
if query.Get("f_E") == "" {
if experienceLevels := linkedInExperienceLevels(searchCriteria); experienceLevels != "" {
query.Set("f_E", experienceLevels)
}
}
if query.Get("f_SB2") == "" {
if salaryBucket := linkedInSalaryBucket(searchCriteria); salaryBucket != "" {
query.Set("f_SB2", salaryBucket)
}
}
u.RawQuery = query.Encode()
default:
return []string{u.String()}
}
var urls []string
if searchKey != "" {
urls = siteSearchURLsWithQueryValues(u, searchKey, searches)
} else {
urls = []string{u.String()}
}
if isIndeedHost(u.Hostname()) {
for _, companyURL := range indeedCompanySearchURLs(company) {
urls = appendUniqueString(urls, companyURL)
}
}
return urls
}

func siteSearchURLsWithQueryValues(base *url.URL, key string, values []string) []string {
if base == nil {
return nil
Expand Down Expand Up @@ -962,6 +1064,69 @@ func targetedSiteSearchQueries(criteria *CriteriaConfig) []string {
return queries
}

func companyTargetSearchQueries(company companyFetchScope, criteria *CriteriaConfig, matchCriteria bool) []string {
names := company.names()
if len(names) == 0 {
return nil
}
domain := company.websiteDomain()
baseQueries := make([]string, 0, len(names))
for _, name := range names {
query := strings.TrimSpace(name)
if domain != "" {
query = strings.TrimSpace(query + " " + domain)
}
baseQueries = appendUniqueString(baseQueries, query)
}
if !matchCriteria {
return baseQueries
}
titleQueries := targetedSiteSearchQueries(criteria)
if len(titleQueries) == 0 {
return baseQueries
}
queries := make([]string, 0)
seen := make(map[string]bool)
for _, baseQuery := range baseQueries {
for _, titleQuery := range titleQueries {
query := strings.TrimSpace(baseQuery + " " + strings.TrimSpace(titleQuery))
if query == "" {
continue
}
key := strings.ToLower(query)
if seen[key] {
continue
}
seen[key] = true
queries = append(queries, query)
}
}
if len(queries) == 0 {
return baseQueries
}
return queries
}

func indeedCompanySearchURLs(company companyFetchScope) []string {
names := company.names()
if len(names) == 0 {
return nil
}
urls := make([]string, 0, len(names))
for _, name := range names {
u := url.URL{
Scheme: "https",
Host: "www.indeed.com",
Path: "/companies/search",
}
query := u.Query()
query.Set("q", name)
u.RawQuery = query.Encode()
urls = appendUniqueString(urls, u.String())
}
return urls
}

func combinedTitleSearchQuery(prefix string, title string) string {
prefix = strings.TrimSpace(prefix)
title = strings.TrimSpace(title)
Expand Down
30 changes: 30 additions & 0 deletions internal/fetcher/site_search_browser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,36 @@ func TestSiteSearchURLForCriteriaBuildsAggregatorSearchURLs(t *testing.T) {
}
}

func TestSiteSearchURLsForCompanyBuildsCompanyQueries(t *testing.T) {
criteria := &CriteriaConfig{}
criteria.Candidate.City = "Seattle"
criteria.Candidate.State = "WA"
criteria.Candidate.CountryCode = "US"
criteria.Filters.TitleRequires = []string{"Senior"}
criteria.Filters.TitleIncludes = []string{"Software Engineer"}
criteria.Filters.WorkSettings.Remote = true
criteria.Filters.WorkSettings.Hybrid = true

scope := companyFetchScope{Company: "GitHub", Website: "https://github.com"}
got := siteSearchURLsForCompany("https://www.indeed.com/jobs", criteria, scope, true)
want := []string{
"https://www.indeed.com/jobs?l=Seattle%2C+WA&q=GitHub+github.com+Senior+Software+Engineer",
"https://www.indeed.com/companies/search?q=GitHub",
}
if strings.Join(got, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("siteSearchURLsForCompany(match criteria) = %#v; want %#v", got, want)
}

got = siteSearchURLsForCompany("https://www.indeed.com/jobs", criteria, scope, false)
want = []string{
"https://www.indeed.com/jobs?q=GitHub+github.com",
"https://www.indeed.com/companies/search?q=GitHub",
}
if strings.Join(got, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("siteSearchURLsForCompany(all) = %#v; want %#v", got, want)
}
}

func TestSiteSearchURLsForCriteriaBuildsTargetedTitleMatrix(t *testing.T) {
criteria := &CriteriaConfig{}
criteria.Candidate.City = "Seattle"
Expand Down
Loading