mirror of
https://github.com/prdlk/jobyssey.git
synced 2026-11-02 13:12:49 +00:00
feat(tools): add Twenty CRM sync scripts and update .gitignore rules
This commit is contained in:
1 parent
b552b70783
commit
39694ee2e5
4 files changed
+612
-7
No files matched your search
Executable
+211
@@ -0,0 +1,211 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Sync job-opportunity email context into Twenty CRM as Notes.
|
||||||
|
|
||||||
|
Called from the /gather-recent-context command. The command owns the email
|
||||||
|
side (Fastmail MCP: search_email / read_thread) and writes the classified
|
||||||
|
threads to a JSON file; this script owns everything deterministic:
|
||||||
|
|
||||||
|
1. Match each email thread to an opportunity on an ACTIVE loop
|
||||||
|
(stage not in the final set) by normalized company name, breaking ties
|
||||||
|
with role-title token overlap against the opportunity name.
|
||||||
|
2. Build one "Email Context — <Company>" note per matched opportunity with
|
||||||
|
a dated markdown timeline of the thread summaries.
|
||||||
|
3. Dedup by thread id against job_scraper/context_sync.json so a re-run
|
||||||
|
never writes the same conversation twice.
|
||||||
|
4. Apply via Twenty REST (TWENTY_API_KEY) with --apply, or emit
|
||||||
|
job_scraper/context_plan.json for the command to apply through the
|
||||||
|
Twenty MCP session.
|
||||||
|
|
||||||
|
Input emails JSON (written by the command):
|
||||||
|
{"threads": [{"thread_id": "...", "company": "Acme", "role": "Backend Engineer"|null,
|
||||||
|
"date": "YYYY-MM-DD", "from": "recruiter@acme.com",
|
||||||
|
"subject": "...", "summary": "1-3 sentence summary",
|
||||||
|
"signal": "interview|assessment|offer|rejection|scheduling|other"}]}
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 tools/gather_recent_context.py --emails <emails.json> # plan only
|
||||||
|
python3 tools/gather_recent_context.py --emails <emails.json> --apply # plan + REST apply
|
||||||
|
... --snapshot crm.json # offline CRM state instead of REST fetch
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from datetime import date
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from twenty_enrich import norm_company, norm_title
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parent.parent
|
||||||
|
STATE = REPO / "job_scraper" / "context_sync.json"
|
||||||
|
PLAN_OUT = REPO / "job_scraper" / "context_plan.json"
|
||||||
|
|
||||||
|
# Loops still in play: everything before a terminal outcome.
|
||||||
|
ACTIVE_STAGES = {"RANKED", "DRAFTED", "APPLIED", "INTERVIEW", "OFFER"}
|
||||||
|
|
||||||
|
|
||||||
|
def load_state(path: Path = STATE) -> dict:
|
||||||
|
if path.exists():
|
||||||
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
return {"processed_thread_ids": [], "last_run": None}
|
||||||
|
|
||||||
|
|
||||||
|
def match_thread(thread: dict, opportunities: list[dict],
|
||||||
|
company_name_by_id: dict[str, str]) -> dict | None:
|
||||||
|
"""Best active-loop opportunity for a thread, or None.
|
||||||
|
|
||||||
|
Opportunity names are role-only (the ' — Company' suffix is stripped once
|
||||||
|
the company relation is set - see tools/twenty_enrich.py), so the company
|
||||||
|
match goes through ``companyId`` -> Company.name. Among a company's active
|
||||||
|
opportunities, prefer the one whose role name shares the most title tokens
|
||||||
|
with the thread's role (when the thread names one); a company with exactly
|
||||||
|
one active loop matches unconditionally.
|
||||||
|
"""
|
||||||
|
key = norm_company(thread.get("company", ""))
|
||||||
|
if not key:
|
||||||
|
return None
|
||||||
|
candidates = [o for o in opportunities
|
||||||
|
if o.get("stage") in ACTIVE_STAGES
|
||||||
|
and norm_company(company_name_by_id.get(o.get("companyId") or "", "")) == key]
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
if len(candidates) == 1 or not thread.get("role"):
|
||||||
|
return candidates[0]
|
||||||
|
role_tokens = set(norm_title(thread["role"]).split())
|
||||||
|
|
||||||
|
def overlap(o):
|
||||||
|
return len(role_tokens & set(norm_title(o.get("name", "")).split()))
|
||||||
|
|
||||||
|
return max(candidates, key=overlap)
|
||||||
|
|
||||||
|
|
||||||
|
def build_note_markdown(threads: list[dict]) -> str:
|
||||||
|
"""Dated timeline, newest first. Only classified thread data - no bodies."""
|
||||||
|
lines = []
|
||||||
|
for t in sorted(threads, key=lambda t: t.get("date") or "", reverse=True):
|
||||||
|
sig = t.get("signal", "other")
|
||||||
|
lines.append(f"## {t.get('date', 'undated')} — {sig}")
|
||||||
|
lines.append(f"**From:** {t.get('from', 'unknown')} · **Subject:** {t.get('subject', '')}")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(t.get("summary", "").strip())
|
||||||
|
lines.append("")
|
||||||
|
return "\n".join(lines).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def build_plan(threads: list[dict], opportunities: list[dict],
|
||||||
|
companies: list[dict], state: dict) -> dict:
|
||||||
|
"""Notes plan grouped one note per opportunity, dedup'd by thread id."""
|
||||||
|
company_name_by_id = {c["id"]: c.get("name", "") for c in companies}
|
||||||
|
seen_ids = set(state.get("processed_thread_ids", []))
|
||||||
|
fresh = [t for t in threads if t.get("thread_id") and t["thread_id"] not in seen_ids]
|
||||||
|
by_opp: dict[str, dict] = {}
|
||||||
|
unmatched = []
|
||||||
|
for t in fresh:
|
||||||
|
o = match_thread(t, opportunities, company_name_by_id)
|
||||||
|
if o is None:
|
||||||
|
unmatched.append({"thread_id": t["thread_id"],
|
||||||
|
"company": t.get("company"), "subject": t.get("subject")})
|
||||||
|
continue
|
||||||
|
slot = by_opp.setdefault(o["id"], {"opportunity_id": o["id"],
|
||||||
|
"opportunity_name": o.get("name"),
|
||||||
|
"company": company_name_by_id.get(o.get("companyId") or "", ""),
|
||||||
|
"threads": []})
|
||||||
|
slot["threads"].append(t)
|
||||||
|
|
||||||
|
notes = []
|
||||||
|
for slot in by_opp.values():
|
||||||
|
company = slot["company"] or "Unknown"
|
||||||
|
notes.append({
|
||||||
|
"opportunity_id": slot["opportunity_id"],
|
||||||
|
"title": f"Email Context — {company} ({date.today().isoformat()})",
|
||||||
|
"markdown": build_note_markdown(slot["threads"]),
|
||||||
|
"thread_ids": [t["thread_id"] for t in slot["threads"]],
|
||||||
|
})
|
||||||
|
return {"notes": notes, "unmatched_threads": unmatched,
|
||||||
|
"skipped_already_processed": len(threads) - len(fresh)}
|
||||||
|
|
||||||
|
|
||||||
|
def apply_plan(plan: dict, client) -> None:
|
||||||
|
"""Create the notes + note targets via REST, then update dedup state."""
|
||||||
|
for n in plan["notes"]:
|
||||||
|
created = client.create_batch("notes", [{
|
||||||
|
"title": n["title"],
|
||||||
|
"bodyV2": {"markdown": n["markdown"]},
|
||||||
|
"position": "last",
|
||||||
|
}])
|
||||||
|
note_id = created[0]["id"]
|
||||||
|
client.create_batch("noteTargets", [{
|
||||||
|
"noteId": note_id,
|
||||||
|
"targetOpportunityId": n["opportunity_id"],
|
||||||
|
"position": "last",
|
||||||
|
}])
|
||||||
|
n["note_id"] = note_id
|
||||||
|
|
||||||
|
|
||||||
|
def commit_state(plan: dict, state: dict, path: Path = STATE) -> None:
|
||||||
|
done = set(state.get("processed_thread_ids", []))
|
||||||
|
for n in plan["notes"]:
|
||||||
|
done.update(n["thread_ids"])
|
||||||
|
state["processed_thread_ids"] = sorted(done)
|
||||||
|
state["last_run"] = date.today().isoformat()
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(state, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
ap.add_argument("--emails", type=Path,
|
||||||
|
help="classified threads JSON written by /gather-recent-context")
|
||||||
|
ap.add_argument("--snapshot", type=Path,
|
||||||
|
help="CRM opportunities JSON (offline; else fetched via REST)")
|
||||||
|
ap.add_argument("--apply", action="store_true",
|
||||||
|
help="create the notes via Twenty REST (needs TWENTY_API_KEY)")
|
||||||
|
ap.add_argument("--commit-plan", type=Path,
|
||||||
|
help="mark an already-applied plan's thread ids as processed and exit "
|
||||||
|
"(used after the command applies the plan through the Twenty MCP session)")
|
||||||
|
ap.add_argument("--out", type=Path, default=PLAN_OUT, help="plan output path")
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
if args.commit_plan:
|
||||||
|
plan = json.loads(args.commit_plan.read_text(encoding="utf-8"))
|
||||||
|
commit_state(plan, load_state())
|
||||||
|
print(f"state committed for {len(plan['notes'])} notes.")
|
||||||
|
return 0
|
||||||
|
if not args.emails:
|
||||||
|
ap.error("--emails is required unless --commit-plan is given")
|
||||||
|
|
||||||
|
threads = json.loads(args.emails.read_text(encoding="utf-8"))["threads"]
|
||||||
|
client = None
|
||||||
|
if args.snapshot:
|
||||||
|
snap = json.loads(args.snapshot.read_text(encoding="utf-8"))
|
||||||
|
opportunities, companies = snap["opportunities"], snap["companies"]
|
||||||
|
else:
|
||||||
|
from twenty_client import TwentyClient
|
||||||
|
client = TwentyClient()
|
||||||
|
opportunities = client.list_all("opportunities")
|
||||||
|
companies = client.list_all("companies")
|
||||||
|
|
||||||
|
state = load_state()
|
||||||
|
plan = build_plan(threads, opportunities, companies, state)
|
||||||
|
|
||||||
|
if args.apply and plan["notes"]:
|
||||||
|
if client is None:
|
||||||
|
from twenty_client import TwentyClient
|
||||||
|
client = TwentyClient()
|
||||||
|
apply_plan(plan, client)
|
||||||
|
commit_state(plan, state)
|
||||||
|
plan["applied"] = True
|
||||||
|
|
||||||
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.out.write_text(json.dumps(plan, indent=2), encoding="utf-8")
|
||||||
|
print(f"plan -> {args.out}")
|
||||||
|
print(f"notes: {len(plan['notes'])} · unmatched threads: {len(plan['unmatched_threads'])}"
|
||||||
|
f" · already processed: {plan['skipped_already_processed']}")
|
||||||
|
if args.apply:
|
||||||
|
print("applied via REST; state updated." if plan["notes"] else "nothing to apply.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -45,13 +45,18 @@ ALLOWED_PERMISSIONS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Personal-data ignore rules that must never disappear from .gitignore.
|
# Personal-data ignore rules that must never disappear from .gitignore.
|
||||||
|
# Fork policy: documents/ (cv, linkedin, diplomas, references, postings,
|
||||||
|
# sources) is deliberately tracked in git - this repo is a personal fork,
|
||||||
|
# not the upstream template. Only generated/derived personal state and
|
||||||
|
# secrets stay ignored.
|
||||||
REQUIRED_IGNORE_RULES = [
|
REQUIRED_IGNORE_RULES = [
|
||||||
"salary_data.json",
|
"salary_data.json",
|
||||||
# Depth-independent: the job-scraper skill resolves `job_scraper/` relative
|
# Depth-independent: the job-scraper skill resolves `job_scraper/` relative
|
||||||
# to its own directory, so the state file lands under .claude/skills/... and
|
# to its own directory, so the state file lands under .claude/skills/... and
|
||||||
# a repo-rooted rule silently fails to match it.
|
# a repo-rooted rule silently fails to match it.
|
||||||
"**/job_scraper/seen_jobs.json",
|
"**/job_scraper/seen_jobs.json",
|
||||||
"**/job_scraper/notion_sync.json",
|
"**/job_scraper/twenty_sync.json",
|
||||||
|
"**/job_scraper/context_sync.json",
|
||||||
"**/job_scraper/*.md",
|
"**/job_scraper/*.md",
|
||||||
"*_BehavioralReport.pdf",
|
"*_BehavioralReport.pdf",
|
||||||
"linkedin_Profile.pdf",
|
"linkedin_Profile.pdf",
|
||||||
@@ -62,15 +67,9 @@ REQUIRED_IGNORE_RULES = [
|
|||||||
"cover_letters/cover_*.*",
|
"cover_letters/cover_*.*",
|
||||||
# /apply also recognizes the uppercase Cover_* naming variant.
|
# /apply also recognizes the uppercase Cover_* naming variant.
|
||||||
"cover_letters/Cover_*.*",
|
"cover_letters/Cover_*.*",
|
||||||
"documents/cv/**",
|
|
||||||
"documents/linkedin/**",
|
|
||||||
"documents/diplomas/**",
|
|
||||||
"documents/references/**",
|
|
||||||
"documents/applications/**",
|
"documents/applications/**",
|
||||||
"documents/postings/**",
|
|
||||||
"documents/interview/**",
|
"documents/interview/**",
|
||||||
"job_search_tracker.csv",
|
"job_search_tracker.csv",
|
||||||
"gmail_sync/",
|
|
||||||
"reports/",
|
"reports/",
|
||||||
"upskill/*.md",
|
"upskill/*.md",
|
||||||
# Depth-independent twin of the rule above. The upskill *skill* resolves
|
# Depth-independent twin of the rule above. The upskill *skill* resolves
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
"""Minimal Twenty CRM REST client (stdlib only).
|
||||||
|
|
||||||
|
Shared by tools/twenty_enrich.py and tools/gather_recent_context.py.
|
||||||
|
|
||||||
|
Auth: set TWENTY_API_KEY (workspace API key from Settings -> API & Webhooks)
|
||||||
|
and optionally TWENTY_BASE_URL (default: https://crm.prad.nu).
|
||||||
|
|
||||||
|
Both consumer scripts also run keyless: they compute a *plan* JSON from local
|
||||||
|
data plus an optional CRM snapshot, and the /sync-twenty or
|
||||||
|
/gather-recent-context command applies the plan through the Twenty MCP
|
||||||
|
server, which is session-authenticated via OAuth. The REST path exists so the
|
||||||
|
whole loop can also run headless (cron, CI) with an API key.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
DEFAULT_BASE_URL = "https://crm.prad.nu"
|
||||||
|
PAGE_LIMIT = 60 # Twenty REST caps page size at 60
|
||||||
|
|
||||||
|
|
||||||
|
class TwentyError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class TwentyClient:
|
||||||
|
def __init__(self, base_url: str | None = None, api_key: str | None = None):
|
||||||
|
self.base_url = (base_url or os.environ.get("TWENTY_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
|
||||||
|
self.api_key = api_key or os.environ.get("TWENTY_API_KEY")
|
||||||
|
if not self.api_key:
|
||||||
|
raise TwentyError(
|
||||||
|
"TWENTY_API_KEY is not set. Either export it, or run without --apply "
|
||||||
|
"to emit a plan JSON and apply it through the Twenty MCP session."
|
||||||
|
)
|
||||||
|
|
||||||
|
def _request(self, method: str, path: str, body: dict | None = None,
|
||||||
|
params: dict | None = None, retries: int = 3):
|
||||||
|
url = f"{self.base_url}{path}"
|
||||||
|
if params:
|
||||||
|
url += "?" + urllib.parse.urlencode(params)
|
||||||
|
data = json.dumps(body).encode() if body is not None else None
|
||||||
|
last_err = None
|
||||||
|
for attempt in range(retries):
|
||||||
|
req = urllib.request.Request(url, data=data, method=method, headers={
|
||||||
|
"Authorization": f"Bearer {self.api_key}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
})
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||||
|
return json.loads(resp.read().decode())
|
||||||
|
except urllib.error.HTTPError as e: # 4xx never retried: same input, same answer
|
||||||
|
detail = e.read().decode(errors="replace")[:500]
|
||||||
|
if e.code == 429 or e.code >= 500:
|
||||||
|
last_err = TwentyError(f"{method} {path} -> {e.code}: {detail}")
|
||||||
|
time.sleep(1.5 * (attempt + 1))
|
||||||
|
continue
|
||||||
|
raise TwentyError(f"{method} {path} -> {e.code}: {detail}") from e
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
last_err = TwentyError(f"{method} {path} -> {e.reason}")
|
||||||
|
time.sleep(1.5 * (attempt + 1))
|
||||||
|
raise last_err
|
||||||
|
|
||||||
|
# -- records ------------------------------------------------------------
|
||||||
|
|
||||||
|
def list_all(self, plural: str, order_by: str = "id") -> list[dict]:
|
||||||
|
"""Every record of an object, cursor-paged."""
|
||||||
|
out: list[dict] = []
|
||||||
|
cursor = None
|
||||||
|
while True:
|
||||||
|
params = {"limit": PAGE_LIMIT, "order_by": order_by}
|
||||||
|
if cursor:
|
||||||
|
params["starting_after"] = cursor
|
||||||
|
payload = self._request("GET", f"/rest/{plural}", params=params)
|
||||||
|
page = payload.get("data", {}).get(plural, [])
|
||||||
|
out.extend(page)
|
||||||
|
page_info = payload.get("pageInfo", {})
|
||||||
|
if not page_info.get("hasNextPage") or not page:
|
||||||
|
break
|
||||||
|
cursor = page_info.get("endCursor") or page[-1]["id"]
|
||||||
|
return out
|
||||||
|
|
||||||
|
def create_batch(self, plural: str, records: list[dict]) -> list[dict]:
|
||||||
|
"""Create records in chunks of PAGE_LIMIT via the batch endpoint."""
|
||||||
|
created: list[dict] = []
|
||||||
|
for i in range(0, len(records), PAGE_LIMIT):
|
||||||
|
payload = self._request("POST", f"/rest/batch/{plural}",
|
||||||
|
body=records[i:i + PAGE_LIMIT])
|
||||||
|
created.extend(payload.get("data", {}).get(plural, []))
|
||||||
|
return created
|
||||||
|
|
||||||
|
def update_one(self, plural: str, record_id: str, patch: dict) -> dict:
|
||||||
|
payload = self._request("PATCH", f"/rest/{plural}/{record_id}", body=patch)
|
||||||
|
data = payload.get("data", {})
|
||||||
|
return next(iter(data.values())) if data else payload
|
||||||
Executable
+295
@@ -0,0 +1,295 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Enrich Twenty CRM opportunities and companies from local job-search data.
|
||||||
|
|
||||||
|
Encodes the deterministic layers of the /sync-twenty enrichment pipeline:
|
||||||
|
|
||||||
|
Opportunity salary (``amount``), in source priority order:
|
||||||
|
1. seen_jobs.json ``salary_max_stated`` (stated in the posting at rank time)
|
||||||
|
2. jobs.db postings salary_max (same company+title, stated on an ATS board)
|
||||||
|
3. jobs.db companies levels_swe_median (levels.fyi company median - labeled estimate)
|
||||||
|
|
||||||
|
Company fields (``domainName``, ``linkedinLink``, ``address``, ``annualRevenue``):
|
||||||
|
- jobs.db companies row matched on normalized name, homepage domain joined
|
||||||
|
through postings.company_domain. Stringified NaN / empty strings are absent.
|
||||||
|
|
||||||
|
Anything these layers cannot ground is emitted in the plan's ``unresolved``
|
||||||
|
section for the calling command to escalate to web research (scout subagents /
|
||||||
|
Perplexity) - this script never guesses.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 tools/twenty_enrich.py # plan -> job_scraper/enrichment_plan.json
|
||||||
|
python3 tools/twenty_enrich.py --apply # plan + apply via REST (needs TWENTY_API_KEY)
|
||||||
|
python3 tools/twenty_enrich.py --snapshot f.json # offline: CRM state from a JSON snapshot
|
||||||
|
|
||||||
|
Snapshot shape (what /sync-twenty exports through MCP):
|
||||||
|
{"opportunities": [{id, url:{primaryLinkUrl}, amount, closeDate, ...}],
|
||||||
|
"companies": [{id, name, domainName, linkedinLink, address, ...}]}
|
||||||
|
|
||||||
|
The plan JSON is idempotent to apply: every entry names the target record id
|
||||||
|
and only fields that are currently empty.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parent.parent
|
||||||
|
SEEN_JOBS = REPO / "job_scraper" / "seen_jobs.json"
|
||||||
|
JOBS_DB = REPO / "documents" / "postings" / "jobs.db"
|
||||||
|
PLAN_OUT = REPO / "job_scraper" / "enrichment_plan.json"
|
||||||
|
|
||||||
|
_SUFFIXES = re.compile(r"\b(inc|llc|ltd|corp|corporation|co|labs|lab|hq|the)\b")
|
||||||
|
|
||||||
|
|
||||||
|
def norm_company(s: str) -> str:
|
||||||
|
"""Normalized company key: lowercase, punctuation collapsed, suffixes stripped."""
|
||||||
|
s = re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
|
||||||
|
s = _SUFFIXES.sub("", s).strip()
|
||||||
|
return re.sub(r"\s+", " ", s)
|
||||||
|
|
||||||
|
|
||||||
|
def norm_title(s: str) -> str:
|
||||||
|
return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def clean_domain(raw: str | None) -> str | None:
|
||||||
|
"""Bare registrable domain or None. Rejects the stringified-NaN artifact."""
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
d = str(raw).strip().lower()
|
||||||
|
for prefix in ("http://", "https://", "www."):
|
||||||
|
d = d.removeprefix(prefix)
|
||||||
|
d = d.rstrip("/")
|
||||||
|
if not d or d == "nan" or "." not in d:
|
||||||
|
return None
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def _has_link(v) -> bool:
|
||||||
|
return bool(v and v.get("primaryLinkUrl"))
|
||||||
|
|
||||||
|
|
||||||
|
def _has_addr(v) -> bool:
|
||||||
|
return bool(v and (v.get("addressCity") or v.get("addressCountry")))
|
||||||
|
|
||||||
|
|
||||||
|
def _has_amount(v) -> bool:
|
||||||
|
return bool(v and v.get("amountMicros"))
|
||||||
|
|
||||||
|
|
||||||
|
def load_local_sources(seen_path: Path = SEEN_JOBS, db_path: Path = JOBS_DB):
|
||||||
|
"""Ranked jobs plus the jobs.db lookup tables the plan builders consume."""
|
||||||
|
seen = json.loads(seen_path.read_text(encoding="utf-8"))["seen"]
|
||||||
|
ranked = [v for v in seen.values() if v.get("status") == "ranked"]
|
||||||
|
|
||||||
|
db_companies: dict[str, dict] = {}
|
||||||
|
domains: dict[str, str] = {}
|
||||||
|
postings_salary: dict[tuple[str, str], int] = {}
|
||||||
|
levels: dict[str, int] = {}
|
||||||
|
if db_path.exists():
|
||||||
|
con = sqlite3.connect(db_path)
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
for r in con.execute(
|
||||||
|
"SELECT company_id, company_name, linkedin_url, country, state, apollo_city,"
|
||||||
|
" organization_revenue, levels_swe_median_usd, levels_median_comp_usd FROM companies"
|
||||||
|
):
|
||||||
|
key = norm_company(r["company_name"])
|
||||||
|
db_companies.setdefault(key, dict(r))
|
||||||
|
lv = r["levels_swe_median_usd"] or r["levels_median_comp_usd"]
|
||||||
|
if lv and lv > 0:
|
||||||
|
levels.setdefault(key, int(lv))
|
||||||
|
for r in con.execute(
|
||||||
|
"SELECT company_id, company_domain, COUNT(*) n FROM postings"
|
||||||
|
" WHERE company_domain IS NOT NULL AND company_domain != ''"
|
||||||
|
" GROUP BY company_id, company_domain ORDER BY n"
|
||||||
|
):
|
||||||
|
domains[r["company_id"]] = r["company_domain"] # last row = most frequent
|
||||||
|
for r in con.execute(
|
||||||
|
"SELECT company_name, title, salary_max, salary_currency FROM postings"
|
||||||
|
" WHERE salary_max IS NOT NULL AND salary_max > 0"
|
||||||
|
):
|
||||||
|
if (r["salary_currency"] or "USD").upper() in ("USD", "$", ""):
|
||||||
|
key = (norm_company(r["company_name"]), norm_title(r["title"]))
|
||||||
|
postings_salary.setdefault(key, int(float(r["salary_max"])))
|
||||||
|
con.close()
|
||||||
|
return ranked, db_companies, domains, postings_salary, levels
|
||||||
|
|
||||||
|
|
||||||
|
def build_salary_plan(ranked, opportunities, postings_salary, levels):
|
||||||
|
"""(updates, unresolved): amount patches for opportunities missing one."""
|
||||||
|
by_url = {}
|
||||||
|
for o in opportunities:
|
||||||
|
u = (o.get("url") or {}).get("primaryLinkUrl")
|
||||||
|
if u:
|
||||||
|
by_url[u] = o
|
||||||
|
updates, unresolved = [], []
|
||||||
|
for v in ranked:
|
||||||
|
o = by_url.get(v.get("url"))
|
||||||
|
if not o or _has_amount(o.get("amount")):
|
||||||
|
continue
|
||||||
|
amount = source = None
|
||||||
|
if v.get("salary_max_stated"):
|
||||||
|
amount, source = int(v["salary_max_stated"]), "seen_jobs.salary_max_stated"
|
||||||
|
else:
|
||||||
|
key = (norm_company(v.get("company", "")), norm_title(v.get("title", "")))
|
||||||
|
if key in postings_salary:
|
||||||
|
amount, source = postings_salary[key], "jobs.db postings"
|
||||||
|
elif norm_company(v.get("company", "")) in levels:
|
||||||
|
amount = levels[norm_company(v.get("company", ""))]
|
||||||
|
source = "levels.fyi company SWE median (estimate)"
|
||||||
|
if amount:
|
||||||
|
updates.append({
|
||||||
|
"id": o["id"],
|
||||||
|
"source": source,
|
||||||
|
"patch": {"amount": {"amountMicros": amount * 1_000_000, "currencyCode": "USD"}},
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
unresolved.append({"id": o["id"], "url": v["url"],
|
||||||
|
"company": v.get("company"), "title": v.get("title")})
|
||||||
|
return updates, unresolved
|
||||||
|
|
||||||
|
|
||||||
|
def build_company_plan(companies, db_companies, domains):
|
||||||
|
"""(updates, unresolved): fill ONLY empty company fields from jobs.db."""
|
||||||
|
updates, unresolved = [], []
|
||||||
|
for c in companies:
|
||||||
|
key = norm_company(c.get("name", ""))
|
||||||
|
m = db_companies.get(key) or {}
|
||||||
|
patch = {}
|
||||||
|
if not _has_link(c.get("domainName")):
|
||||||
|
dom = clean_domain(domains.get(m.get("company_id", "")))
|
||||||
|
if dom:
|
||||||
|
patch["domainName"] = {"primaryLinkUrl": f"https://{dom}",
|
||||||
|
"primaryLinkLabel": c["name"]}
|
||||||
|
if not _has_link(c.get("linkedinLink")):
|
||||||
|
li = (m.get("linkedin_url") or "").strip()
|
||||||
|
if li:
|
||||||
|
patch["linkedinLink"] = {"primaryLinkUrl": li.replace("http://", "https://"),
|
||||||
|
"primaryLinkLabel": c["name"]}
|
||||||
|
if not _has_addr(c.get("address")):
|
||||||
|
addr = {k: v for k, v in [("addressCity", m.get("apollo_city")),
|
||||||
|
("addressState", m.get("state")),
|
||||||
|
("addressCountry", m.get("country"))] if v}
|
||||||
|
if addr:
|
||||||
|
patch["address"] = addr
|
||||||
|
if patch:
|
||||||
|
updates.append({"id": c["id"], "name": c["name"], "patch": patch})
|
||||||
|
# Fields still empty after the local layer -> web-research escalation
|
||||||
|
missing = []
|
||||||
|
if not _has_link(c.get("domainName")) and "domainName" not in patch:
|
||||||
|
missing.append("domain")
|
||||||
|
if not _has_link(c.get("linkedinLink")) and "linkedinLink" not in patch:
|
||||||
|
missing.append("linkedin")
|
||||||
|
if not _has_addr(c.get("address")) and "address" not in patch:
|
||||||
|
missing.append("address")
|
||||||
|
if missing:
|
||||||
|
unresolved.append({"id": c["id"], "name": c["name"], "missing": missing})
|
||||||
|
return updates, unresolved
|
||||||
|
|
||||||
|
|
||||||
|
def build_name_cleanup_plan(opportunities):
|
||||||
|
"""Strip the ' — <Company>' suffix from opportunity names.
|
||||||
|
|
||||||
|
Runs LAST, and only for opportunities whose company relation is already
|
||||||
|
set: the suffix is the human-readable fallback while the record is
|
||||||
|
unlinked, and becomes redundant once ``companyId`` points at the Company.
|
||||||
|
An unlinked opportunity keeps its suffixed name so it stays identifiable.
|
||||||
|
"""
|
||||||
|
updates = []
|
||||||
|
for o in opportunities:
|
||||||
|
name = o.get("name") or ""
|
||||||
|
if o.get("companyId") and " — " in name:
|
||||||
|
role = name.split(" — ")[0].strip()
|
||||||
|
if role and role != name:
|
||||||
|
updates.append({"id": o["id"], "patch": {"name": role}})
|
||||||
|
return updates
|
||||||
|
|
||||||
|
|
||||||
|
def coverage(opportunities, companies) -> dict:
|
||||||
|
n, m = len(companies), len(opportunities)
|
||||||
|
return {
|
||||||
|
"companies": {
|
||||||
|
"total": n,
|
||||||
|
"domain": sum(1 for c in companies if _has_link(c.get("domainName"))),
|
||||||
|
"linkedin": sum(1 for c in companies if _has_link(c.get("linkedinLink"))),
|
||||||
|
"address": sum(1 for c in companies if _has_addr(c.get("address"))),
|
||||||
|
},
|
||||||
|
"opportunities": {
|
||||||
|
"total": m,
|
||||||
|
"salary": sum(1 for o in opportunities if _has_amount(o.get("amount"))),
|
||||||
|
"company_link": sum(1 for o in opportunities if o.get("companyId")),
|
||||||
|
"score": sum(1 for o in opportunities if o.get("score") is not None),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
ap.add_argument("--snapshot", type=Path,
|
||||||
|
help="CRM state JSON (offline mode; else fetched via REST)")
|
||||||
|
ap.add_argument("--apply", action="store_true",
|
||||||
|
help="apply the plan via Twenty REST (needs TWENTY_API_KEY)")
|
||||||
|
ap.add_argument("--out", type=Path, default=PLAN_OUT, help="plan output path")
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
ranked, db_companies, domains, postings_salary, levels = load_local_sources()
|
||||||
|
|
||||||
|
client = None
|
||||||
|
if args.snapshot:
|
||||||
|
snap = json.loads(args.snapshot.read_text(encoding="utf-8"))
|
||||||
|
opportunities, companies = snap["opportunities"], snap["companies"]
|
||||||
|
else:
|
||||||
|
from twenty_client import TwentyClient # noqa: deferred import keeps offline mode key-free
|
||||||
|
client = TwentyClient()
|
||||||
|
opportunities = client.list_all("opportunities")
|
||||||
|
companies = client.list_all("companies")
|
||||||
|
|
||||||
|
sal_updates, sal_unresolved = build_salary_plan(ranked, opportunities, postings_salary, levels)
|
||||||
|
comp_updates, comp_unresolved = build_company_plan(companies, db_companies, domains)
|
||||||
|
name_updates = build_name_cleanup_plan(opportunities)
|
||||||
|
|
||||||
|
plan = {
|
||||||
|
"coverage_before": coverage(opportunities, companies),
|
||||||
|
"opportunity_updates": sal_updates,
|
||||||
|
"company_updates": comp_updates,
|
||||||
|
"name_cleanup": name_updates,
|
||||||
|
"unresolved": {
|
||||||
|
"opportunities_without_salary": sal_unresolved,
|
||||||
|
"companies_missing_fields": comp_unresolved,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if args.apply:
|
||||||
|
if client is None:
|
||||||
|
from twenty_client import TwentyClient
|
||||||
|
client = TwentyClient()
|
||||||
|
for u in sal_updates:
|
||||||
|
client.update_one("opportunities", u["id"], u["patch"])
|
||||||
|
for u in comp_updates:
|
||||||
|
client.update_one("companies", u["id"], u["patch"])
|
||||||
|
# Name cleanup LAST: it must observe the post-link state.
|
||||||
|
for u in name_updates:
|
||||||
|
client.update_one("opportunities", u["id"], u["patch"])
|
||||||
|
plan["applied"] = {"opportunities": len(sal_updates), "companies": len(comp_updates),
|
||||||
|
"names_cleaned": len(name_updates)}
|
||||||
|
|
||||||
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.out.write_text(json.dumps(plan, indent=2), encoding="utf-8")
|
||||||
|
cov = plan["coverage_before"]
|
||||||
|
print(f"plan -> {args.out}")
|
||||||
|
print(f"opportunity salary updates: {len(sal_updates)} "
|
||||||
|
f"(coverage {cov['opportunities']['salary']}/{cov['opportunities']['total']} before)")
|
||||||
|
print(f"company field updates: {len(comp_updates)} · name cleanups: {len(name_updates)}")
|
||||||
|
print(f"unresolved -> web research: {len(sal_unresolved)} postings, "
|
||||||
|
f"{len(comp_unresolved)} companies")
|
||||||
|
if args.apply:
|
||||||
|
print("applied via REST.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Reference in new issue
Block a user