Gratis Gap Scanner Vind momentum aandelen in real-time

// ─────────────────────────────────────────────────────────────────────────────
// Gap Scanner Worker — FMP (gainers) + Finnhub company news + RSS feeds
//
// Scan:    FMP /gainers + /actives — ALL US stocks, 2 API calls
// News:    Finnhub /company-news per gapper — direct company news
//          Benzinga RSS + GlobeNewswire RSS + PR Newswire RSS — background feeds
// Market:  Finnhub (SPY, QQQ, VIX)
//
// API keys:
//   ?key=FINNHUB_KEY  — news + market data
//   ?fmp=FMP_KEY      — gainers scan
// ─────────────────────────────────────────────────────────────────────────────

const CORS = {
  'Access-Control-Allow-Origin': '*',
  'Access-Control-Allow-Methods': 'GET, OPTIONS',
  'Access-Control-Allow-Headers': 'Content-Type',
  'Access-Control-Max-Age': '86400',
};

const FH_BASE  = 'https://finnhub.io/api/v1';
const FMP_BASE_V3     = 'https://financialmodelingprep.com/api/v3';
const FMP_BASE_STABLE = 'https://financialmodelingprep.com/stable';

const RSS_FEEDS = [
  { url: 'https://feeds.benzinga.com/benzinga',                                    source: 'Benzinga' },
  { url: 'https://www.globenewswire.com/RssFeed/industry/9600',                    source: 'GlobeNewswire' },
  { url: 'https://www.prnewswire.com/rss/news-releases-list.rss',                  source: 'PR Newswire' },
  { url: 'https://www.businesswire.com/rss/home/?rss=G1',                          source: 'Business Wire' },
  { url: 'https://www.accesswire.com/rss',                                         source: 'AccessWire' },
];

export default {
  async fetch(request) {
    if (request.method === 'OPTIONS') return new Response(null, { headers: CORS });
    const url = new URL(request.url);
    const key = url.searchParams.get('key') ?? '';
    const fmp = url.searchParams.get('fmp') ?? '';

    if (url.pathname === '/health')   return ok({ ok: true, ts: new Date().toISOString() });
    if (url.pathname === '/scan')     return handleScan(url, key, fmp);
    if (url.pathname === '/market')   return handleMarket(key);
    if (url.pathname === '/news')     return handleNews(key);
    if (url.pathname === '/movers')   return handleMovers(key, fmp);
    if (url.pathname === '/videos')   return handleVideos();
    if (url.pathname === '/debug')    return handleDebug(fmp);
    if (url.pathname === '/earnings') return handleEarnings();
    if (url.pathname === '/sectors')  return handleSectors(key);
    if (url.pathname === '/fda')      return handleFDA();

    return new Response('Gap Scanner Worker: /scan /market /news /videos /health', { headers: CORS });
  }
};

// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────
async function fh(endpoint, key, params = {}) {
  if (!key) throw new Error('Geen Finnhub API key opgegeven.');
  const u = new URL(`${FH_BASE}${endpoint}`);
  u.searchParams.set('token', key);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u.toString(), { headers: { 'User-Agent': 'GapScanner/1.0' } });
  if (res.status === 401) throw new Error('Ongeldige Finnhub API key.');
  if (res.status === 429) throw new Error('Rate limit bereikt. Wacht even en probeer opnieuw.');
  if (!res.ok) throw new Error(`Finnhub fout ${res.status}`);
  return res.json();
}

async function fmpFetch(endpoint, fmpKey, useStable = false) {
  if (!fmpKey) throw new Error('Geen FMP API key opgegeven.');
  const base = useStable ? FMP_BASE_STABLE : FMP_BASE_V3;
  const res = await fetch(`${base}${endpoint}?apikey=${fmpKey}`, { headers: { 'User-Agent': 'GapScanner/1.0' } });
  if (res.status === 401 || res.status === 403) throw new Error('Ongeldige FMP API key.');
  if (res.status === 429) throw new Error('FMP rate limit bereikt. Je hebt 250 gratis calls per dag.');
  if (!res.ok) throw new Error(`FMP fout ${res.status}`);
  const data = await res.json();

  // FMP can return:
  //   plain array:                     [ {symbol,price,...}, ... ]
  //   wrapped array:                   { mostGainerStock: [...] }
  //   wrapped array v4:                { data: [...] }
  //   error object:                    { "Error Message": "..." }
  //   empty object on weekend/holiday: {}

  if (Array.isArray(data)) return data;

  // Check for error message
  if (data['Error Message'] || data.error || data.message) {
    const msg = data['Error Message'] || data.error || data.message;
    throw new Error(`FMP: ${msg}`);
  }

  // Find the first array value in the object
  for (const val of Object.values(data)) {
    if (Array.isArray(val) && val.length > 0) return val;
  }

  // Return empty if nothing found (weekend, holiday, etc.)
  return [];
}

// Normalise a single FMP stock object — field names vary between endpoint versions
function normFMP(s) {
  const ticker    = (s.symbol ?? s.ticker ?? '').toUpperCase().trim();
  // changesPercentage can be "5.23%" (string) or 5.23 (number)
  const rawPct    = s.changesPercentage ?? s.changePercentage ?? s.change_percentage ?? 0;
  const changePct = parseFloat(String(rawPct).replace('%','')) || 0;
  const price     = parseFloat(s.price ?? s.currentPrice ?? 0) || 0;
  const volume    = parseInt(s.volume ?? 0) || null;
  const company   = s.companyName ?? s.name ?? s.company ?? '';
  return { ticker, changePct, price, volume, company };
}

// ─────────────────────────────────────────────────────────────────────────────
// /scan — FMP gainers + company-specific news per stock
// ─────────────────────────────────────────────────────────────────────────────
async function handleScan(url, finnhubKey, fmpKey) {
  if (!fmpKey) return err('Geen FMP API key. Voer je FMP key in op de pagina.');
  try {
    const p        = url.searchParams;
    const minGap   = parseFloat(p.get('minGap')   ?? '10');
    const minPrice = parseFloat(p.get('minPrice') ?? '1');
    const maxPrice = parseFloat(p.get('maxPrice') ?? '100');

    // Step 1: Get all gainers + actives from FMP simultaneously
    const [gainersData, activesData] = await Promise.all([
      fmpFetch('/biggest-gainers', fmpKey, true).catch(() => []),
      fmpFetch('/most-actives', fmpKey, true).catch(() => []),
    ]);

    // Merge + deduplicate, normalise FMP field names
    const seen = new Set();
    const allStocks = [];
    for (const s of [...(gainersData || []), ...(activesData || [])]) {
      const n = normFMP(s);
      if (!n.ticker || n.ticker.includes('.') || n.ticker.includes('-') || seen.has(n.ticker)) continue;
      seen.add(n.ticker);
      allStocks.push(n);
    }

    // Step 2: Filter by price + gap criteria first
    const candidates = allStocks.filter(s =>
      s.price >= minPrice && s.price <= maxPrice && s.changePct >= minGap
    );

    if (candidates.length === 0) {
      return ok({ ok: true, count: 0, scanned: allStocks.length, stocks: [], scannedAt: new Date().toISOString() });
    }

    // Step 3: Get today's date range for company news
    const today = new Date();
    const from  = new Date(today); from.setDate(from.getDate() - 1);
    const dateFrom = from.toISOString().slice(0, 10);
    const dateTo   = today.toISOString().slice(0, 10);

    // Sources that publish generic market overviews — blocked for per-stock news
    const BLOCKED_SOURCES = new Set([
      'chartmill','markets insider','simply wall st','simplywall',
      'wsj markets','investing.com','seekingalpha','seeking alpha',
      'thestreet','investopedia','motley fool','fool.com','zacks',
      'marketbeat','stockanalysis','stockanalysis.com','finviz',
    ]);

    // Generic headline patterns to filter out
    const GENERIC_PHRASES = [
      'top stock movements','unusual volume','top gainers','top losers',
      'gapping on','pre-market session','after-hours','market session',
      'stocks to watch','movers today','which stocks','session recap',
      'week ahead','market wrap','opening bell','closing bell',
    ];

    const isRealNews = (headline, source) => {
      const src   = (source   || '').toLowerCase();
      const title = (headline || '').toLowerCase();
      if (BLOCKED_SOURCES.has(src)) return false;
      if (GENERIC_PHRASES.some(p => title.includes(p))) return false;
      return true;
    };

    // Score a news item — higher = more relevant catalyst
    // Limit to top 15 candidates by gap %
    const topCandidates = candidates.slice(0, 15);

    // Fetch all news sources simultaneously for each candidate
    const fetchStockNews = async (stock) => {
      const ticker  = stock.ticker;
      const company = stock.company || '';
      // Short company name for matching (first 2 meaningful words)
      const shortName = company.split(' ').filter(w => w.length > 2).slice(0, 2).join(' ').toLowerCase();

      const [
        finnhubNews,
        yahooNews,
        secFilings,
        googleNews,
      ] = await Promise.all([
        // Finnhub company news
        finnhubKey
          ? fh('/company-news', finnhubKey, { symbol: ticker, from: dateFrom, to: dateTo })
              .catch(() => [])
          : Promise.resolve([]),
        // Yahoo Finance RSS per ticker
        fetchYahooRSS(ticker),
        // SEC EDGAR 8-K filings for this ticker
        fetchSECFilings(ticker),
        // Google News RSS — free, no key, all sources
        fetchGoogleNews(ticker, company),
      ]);

      // Normalise all sources into one format
      const allNews = [];

      // Finnhub news
      for (const n of (finnhubNews || [])) {
        if (!isRealNews(n.headline, n.source)) continue;
        allNews.push({
          title:   n.headline,
          link:    n.url,
          source:  n.source || 'Finnhub',
          pubDate: n.datetime ? new Date(n.datetime * 1000).toISOString() : null,
          datetime: n.datetime || 0,
        });
      }

      // Yahoo Finance RSS
      for (const n of (yahooNews || [])) {
        if (!isRealNews(n.title, n.source)) continue;
        allNews.push({ ...n, datetime: n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0 });
      }

      // SEC EDGAR 8-K filings — always real, never filtered
      for (const n of (secFilings || [])) {
        allNews.push({ ...n, datetime: n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0 });
      }

      // Google News — free, no key, catches everything else
      for (const n of (googleNews || [])) {
        if (!isRealNews(n.title, n.source)) continue;
        allNews.push({ ...n, datetime: n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0 });
      }

      // Also check RSS items for this ticker or company name
      const rssForStock = rssItems.filter(item => {
        if ((item.tickers || []).includes(ticker)) return true;
        if (shortName && item.title.toLowerCase().includes(shortName)) return true;
        return false;
      });
      for (const n of rssForStock) {
        if (!isRealNews(n.title, n.source)) continue;
        allNews.push({
          title:   n.title,
          link:    n.link,
          source:  n.source,
          pubDate: n.pubDate,
          datetime: n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0,
        });
      }

      // Deduplicate by title similarity
      const seen = new Set();
      const unique = allNews.filter(n => {
        const key = (n.title || '').slice(0, 60).toLowerCase();
        if (seen.has(key)) return false;
        seen.add(key);
        return true;
      });

      // Sort by score (source quality + catalyst keywords + recency)
      unique.sort((a, b) => newsScore(b) - newsScore(a));

      return { ticker, news: unique.slice(0, 5) };
    };

    // Fetch RSS feeds first, then use them inside fetchStockNews
    const rssItems = await fetchAllRSS();

    // Fetch all stock news simultaneously
    const stockNewsResults = await Promise.all(
      topCandidates.map(fetchStockNews)
    );

    // Build news map
    const companyNewsMap = {};
    for (const { ticker, news } of stockNewsResults) {
      if (!news.length) continue;
      companyNewsMap[ticker] = {
        title:   news[0].title,
        link:    news[0].link,
        source:  news[0].source,
        pubDate: news[0].pubDate,
        allNews: news,
      };
    }

    // Step 5: Build final stock list (s already normalised by normFMP)
    const stocks = candidates.map(s => {
      const news = companyNewsMap[s.ticker];
      return {
        ticker:     s.ticker,
        company:    s.company,
        price:      +s.price.toFixed(2),
        prevClose:  s.price > 0 && s.changePct !== 0
                      ? +(s.price / (1 + s.changePct / 100)).toFixed(2)
                      : null,
        gapPct:     +s.changePct.toFixed(2),
        volume:     s.volume,
        relVol:     null,
        hasNews:    !!news,
        headline:   news?.title    ?? null,
        newsUrl:    news?.link     ?? null,
        newsSource: news?.source   ?? null,
        allNews:    news?.allNews  ?? [],
        score:      calcScore(s.changePct, null, s.price, !!news),
      };
    });

    // Sort by gap % high to low, take top 10 (same as movers — keeps columns equal height)
    stocks.sort((a, b) => b.gapPct - a.gapPct);
    const top15 = stocks.slice(0, 10);

    return ok({
      ok:        true,
      count:     top15.length,
      scanned:   allStocks.length,
      source:    'FMP gainers (alle US aandelen)',
      stocks:    top15,
      scannedAt: new Date().toISOString(),
    });

  } catch(e) {
    return err(e.message);
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// /debug — shows raw FMP response so we can diagnose field names
// ─────────────────────────────────────────────────────────────────────────────
async function handleDebug(fmpKey) {
  if (!fmpKey) return err('Geen FMP key');
  try {
    const res  = await fetch(`${FMP_BASE_STABLE}/biggest-gainers?apikey=${fmpKey}`);
    const text = await res.text();
    let parsed;
    try { parsed = JSON.parse(text); } catch(e) { parsed = null; }

    const isArray    = Array.isArray(parsed);
    const topKeys    = parsed && !isArray ? Object.keys(parsed) : [];
    const firstItem  = isArray ? parsed[0] : (parsed && topKeys.length ? parsed[topKeys[0]]?.[0] : parsed);
    const firstKeys  = firstItem ? Object.keys(firstItem) : [];

    return ok({
      ok:           true,
      httpStatus:   res.status,
      isArray,
      count:        isArray ? parsed.length : '?',
      topLevelKeys: topKeys,
      firstItemKeys: firstKeys,
      firstItem,
      rawPreview:   text.slice(0, 300),
    });
  } catch(e) { return err(e.message); }
}

// ─────────────────────────────────────────────────────────────────────────────
// /movers — Yesterday's top gainers + losers with context
// Uses FMP /gainers, /losers + Finnhub company news per stock
// ─────────────────────────────────────────────────────────────────────────────
async function handleMovers(finnhubKey, fmpKey) {
  if (!fmpKey) return err('Geen FMP API key.');
  try {
    // Fetch gainers + losers simultaneously
    const [gainers, losers] = await Promise.all([
      fmpFetch('/biggest-gainers', fmpKey, true).catch(() => []),
      fmpFetch('/biggest-losers',  fmpKey, true).catch(() => []),
    ]);

    // Date range for news (yesterday + today)
    const today = new Date();
    const from  = new Date(today); from.setDate(from.getDate() - 2);
    const dateFrom = from.toISOString().slice(0, 10);
    const dateTo   = today.toISOString().slice(0, 10);

    // Enrich top 10 gainers and top 10 losers with multi-source news
    const enrichStocks = async (rawStocks, limit = 10) => {
      const top = (rawStocks || [])
        .map(normFMP)
        .filter(s => s.ticker && !s.ticker.includes('.') && !s.ticker.includes('-'))
        .slice(0, limit);

      const newsResults = await Promise.all(
        top.map(async s => {
          const [finnhubNews, yahooNews, secFilings, googleNews] = await Promise.all([
            finnhubKey
              ? fh('/company-news', finnhubKey, { symbol: s.ticker, from: dateFrom, to: dateTo })
                  .catch(() => [])
              : Promise.resolve([]),
            fetchYahooRSS(s.ticker),
            fetchSECFilings(s.ticker),
            fetchGoogleNews(s.ticker, s.company),
          ]);

          const allNews = [];
          const seen    = new Set();

          // Finnhub
          for (const n of (finnhubNews || [])) {
            const key = (n.headline || '').slice(0, 60).toLowerCase();
            if (seen.has(key)) continue;
            seen.add(key);
            allNews.push({
              title:   n.headline,
              link:    n.url,
              source:  n.source || 'Finnhub',
              pubDate: n.datetime ? new Date(n.datetime*1000).toISOString() : null,
              datetime: n.datetime || 0,
            });
          }
          // Yahoo
          for (const n of (yahooNews || [])) {
            const key = (n.title || '').slice(0, 60).toLowerCase();
            if (seen.has(key)) continue;
            seen.add(key);
            allNews.push({ ...n, datetime: n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0 });
          }
          // SEC
          for (const n of (secFilings || [])) {
            allNews.push({ ...n, datetime: n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0 });
          }
          // Google News
          for (const n of (googleNews || [])) {
            const key = (n.title || '').slice(0, 60).toLowerCase();
            if (seen.has(key)) continue;
            seen.add(key);
            allNews.push({ ...n, datetime: n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0 });
          }

          allNews.sort((a, b) => newsScore(b) - newsScore(a));
          return { ticker: s.ticker, news: allNews.slice(0, 3) };
        })
      );

      const newsMap = {};
      for (const { ticker, news } of newsResults) newsMap[ticker] = news;

      return top.map(s => {
        const news = newsMap[s.ticker] ?? [];
        return {
          ticker:    s.ticker,
          company:   s.company,
          price:     +s.price.toFixed(2),
          changePct: +s.changePct.toFixed(2),
          prevClose: s.price > 0 && s.changePct !== 0
                       ? +(s.price / (1 + s.changePct / 100)).toFixed(2)
                       : null,
          volume:    s.volume,
          hasNews:   news.length > 0,
          news:      news.map(n => ({
            title:   n.title,
            link:    n.link,
            source:  n.source,
            pubDate: n.pubDate,
          })),
        };
      });
    };

    const [enrichedGainers, enrichedLosers] = await Promise.all([
      enrichStocks(gainers, 10),
      enrichStocks(losers,  10),
    ]);

    return ok({
      ok:      true,
      gainers: enrichedGainers,
      losers:  enrichedLosers,
      ts:      new Date().toISOString(),
    });

  } catch(e) {
    return err(e.message);
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// /market — SPY, QQQ, VIX
// ─────────────────────────────────────────────────────────────────────────────
async function handleMarket(key) {
  if (!key) return err('Geen Finnhub API key.');
  try {
    const [spy, qqq, vix] = await Promise.all([
      fh('/quote', key, { symbol: 'SPY' }),
      fh('/quote', key, { symbol: 'QQQ' }),
      fh('/quote', key, { symbol: 'VIX' }).catch(() =>
        fh('/quote', key, { symbol: '^VIX' }).catch(() => null)
      ),
    ]);
    const fmt = (q, sym) => q ? {
      symbol: sym, price: q.c,
      changePct: q.pc > 0 ? +((q.c - q.pc) / q.pc * 100).toFixed(2) : null,
      prevClose: q.pc, high: q.h, low: q.l,
    } : { symbol: sym, error: 'Geen data' };
    return ok({ ok: true, spy: fmt(spy,'SPY'), qqq: fmt(qqq,'QQQ'), vix: fmt(vix,'VIX'), ts: new Date().toISOString() });
  } catch(e) { return err(e.message); }
}

// ─────────────────────────────────────────────────────────────────────────────
// /news — General news feed (Finnhub + RSS, smart ranked)
// Used for the news panel. Scan results have their own company news.
// ─────────────────────────────────────────────────────────────────────────────
async function handleNews(key) {
  try {
    const [finnhubNews, rssItems] = await Promise.all([
      key ? fh('/news', key, { category: 'general' }).catch(() => []) : Promise.resolve([]),
      fetchAllRSS(),
    ]);

    const raw = [];
    for (const n of (finnhubNews || []).slice(0, 20))
      raw.push({ title: n.headline, link: n.url, source: n.source ?? 'Finnhub', pubDate: n.datetime ? new Date(n.datetime*1000).toISOString() : null, tickers: n.related ? n.related.split(',').map(t=>t.trim()).filter(Boolean) : [] });
    for (const r of rssItems.slice(0, 60))
      raw.push({ title: r.title, link: r.link, source: r.source, pubDate: r.pubDate, tickers: r.tickers ?? [] });

    const CATALYST = ['fda','approval','approved','clearance','earnings','revenue','beat','beats','guidance','merger','acquisition','acquires','contract','award','partnership','deal','ipo','offering','uplisting','clinical','trial','phase','breakthrough','record','surge'];
    const srcScore = s => {
      s = (s||'').toLowerCase();
      if (s.includes('sec.gov') || s === 'sec edgar')             return 40;
      if (s.includes('businesswire'))                              return 30;
      if (s.includes('accesswire'))                                return 25;
      if (s.includes('globenewswire') || s.includes('globe'))      return 20;
      if (s.includes('pr newswire') || s.includes('prnews'))       return 20;
      if (s.includes('benzinga'))                                  return 15;
      if (s.includes('yahoo'))                                     return 12;
      return 0;
    };
    const ageScore = d => { if(!d)return 0; try{ const m=(Date.now()-new Date(d).getTime())/60000; return m<30?10:m<60?7:m<120?4:1; }catch{return 0;} };
    const catScore = t => { if(!t)return 0; const l=t.toLowerCase(); return CATALYST.some(w=>l.includes(w))?20:0; };

    const scored = raw.map(item => ({...item, _score: srcScore(item.source)+ageScore(item.pubDate)+catScore(item.title)+(item.tickers?.length>0?5:0)}));
    scored.sort((a,b) => b._score!==a._score ? b._score-a._score : new Date(b.pubDate||0)-new Date(a.pubDate||0));
    const items = scored.slice(0,50).map(({_score,...item})=>item);
    return ok({ ok: true, count: items.length, items, ts: new Date().toISOString() });
  } catch(e) { return err(e.message); }
}

// ─────────────────────────────────────────────────────────────────────────────
// /earnings — Earnings news via Benzinga RSS + Google News (free, no key needed)
// Filters for earnings-related headlines from today
// ─────────────────────────────────────────────────────────────────────────────
async function handleEarnings() {
  try {
    const EARNINGS_KEYWORDS = [
      'earnings','quarterly results','q1','q2','q3','q4','revenue','eps',
      'beats','misses','guidance','profit','loss','fiscal','annual results',
      'reports results','financial results','fourth quarter','third quarter',
      'second quarter','first quarter','full year',
    ];

    const isEarningsNews = (title) => {
      const t = (title || '').toLowerCase();
      return EARNINGS_KEYWORDS.some(k => t.includes(k));
    };

    // Fetch earnings-related news from multiple free RSS sources simultaneously
    const [benzinga, googleEarnings, globenews] = await Promise.all([
      // Benzinga RSS — filtered for earnings
      fetchRSS('https://feeds.benzinga.com/benzinga', 'Benzinga'),
      // Google News RSS for "earnings results today"
      (async () => {
        try {
          const url = `https://news.google.com/rss/search?q=earnings+results+today+stock&hl=en-US&gl=US&ceid=US:en`;
          const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
          if (!res.ok) return [];
          return parseRSS(await res.text(), 'Google News');
        } catch { return []; }
      })(),
      // GlobeNewswire — often has official earnings press releases
      fetchRSS('https://www.globenewswire.com/RssFeed/industry/9600', 'GlobeNewswire'),
    ]);

    // Merge, filter for earnings content, deduplicate
    const seen  = new Set();
    const items = [];
    for (const n of [...benzinga, ...googleEarnings, ...globenews]) {
      if (!isEarningsNews(n.title)) continue;
      const key = (n.title || '').slice(0, 60).toLowerCase();
      if (seen.has(key)) continue;
      seen.add(key);
      items.push({
        title:   n.title,
        link:    n.link,
        source:  n.source,
        pubDate: n.pubDate,
        tickers: n.tickers || [],
      });
    }

    // Sort by recency
    items.sort((a, b) => new Date(b.pubDate || 0) - new Date(a.pubDate || 0));

    return ok({ ok: true, count: items.length, items: items.slice(0, 20), ts: new Date().toISOString() });
  } catch(e) { return err(e.message); }
}

// ─────────────────────────────────────────────────────────────────────────────
// /sectors — Sector performance snapshot via FMP stable API
// ─────────────────────────────────────────────────────────────────────────────
// ─────────────────────────────────────────────────────────────────────────────
// /sectors — Sector performance via SPDR Sector ETFs (Finnhub free)
// No FMP needed — uses the Finnhub key the user already has
// ─────────────────────────────────────────────────────────────────────────────
async function handleSectors(finnhubKey) {
  if (!finnhubKey) return err('Geen Finnhub API key voor sectordata.');
  try {
    const SECTOR_ETFS = [
      { ticker: 'XLK',  sector: 'Technology' },
      { ticker: 'XLV',  sector: 'Healthcare' },
      { ticker: 'XLF',  sector: 'Financials' },
      { ticker: 'XLE',  sector: 'Energy' },
      { ticker: 'XLI',  sector: 'Industrials' },
      { ticker: 'XLY',  sector: 'Consumer Discretionary' },
      { ticker: 'XLP',  sector: 'Consumer Staples' },
      { ticker: 'XLB',  sector: 'Materials' },
      { ticker: 'XLRE', sector: 'Real Estate' },
      { ticker: 'XLU',  sector: 'Utilities' },
      { ticker: 'XLC',  sector: 'Communication Services' },
    ];

    // Fetch all sector ETF quotes simultaneously
    const quotes = await Promise.all(
      SECTOR_ETFS.map(s =>
        fh('/quote', finnhubKey, { symbol: s.ticker })
          .then(q => ({
            sector:    s.sector,
            ticker:    s.ticker,
            price:     q.c ?? null,
            changePct: q.pc > 0 ? +((q.c - q.pc) / q.pc * 100).toFixed(2) : 0,
          }))
          .catch(() => ({ sector: s.sector, ticker: s.ticker, price: null, changePct: 0 }))
      )
    );

    const items = quotes
      .filter(q => q.price !== null)
      .sort((a, b) => b.changePct - a.changePct);

    return ok({ ok: true, count: items.length, items, ts: new Date().toISOString() });
  } catch(e) { return err(e.message); }
}

// ─────────────────────────────────────────────────────────────────────────────
// /fda — FDA filings via SEC EDGAR full-text search (free, no key)
// Direct links to actual filing documents via accession number
// ─────────────────────────────────────────────────────────────────────────────
async function handleFDA() {
  try {
    const url = `https://efts.sec.gov/LATEST/search-index?q=%22PDUFA%22+OR+%22FDA+approval%22+OR+%22NDA+submission%22+OR+%22BLA+submission%22&dateRange=custom&startdt=${getDateDaysAgo(14)}&enddt=${getDateDaysAgo(0)}&forms=8-K`;
    const res = await fetch(url, {
      headers: { 'User-Agent': 'GapScanner/1.0 contact@goldminey.com', 'Accept': 'application/json' }
    });
    if (!res.ok) throw new Error(`SEC EDGAR fout ${res.status}`);
    const data = await res.json();
    const hits  = data.hits?.hits ?? [];

    const items = hits.slice(0, 10).map(h => {
      const src = h._source || {};

      // EDGAR display_names: ["Company Name (TICKER) (CIK 0001234567)"]
      const displayNames = src.display_names || [];
      const firstName    = displayNames[0] || '';
      const tickerMatch  = firstName.match(/\(([A-Z]{1,6})\)/);
      const ticker       = tickerMatch ? tickerMatch[1] : '';
      const company      = firstName.split('(')[0].trim();

      // The hit _id is the accession number — use it to build a direct EDGAR filing link
      // Format: 0001234567-26-000123 → direct index page on SEC.gov
      const accessionRaw = (h._id || '').replace(/\//g, '-');
      // CIK is embedded in accession number first 10 digits
      const cik          = src.ciks?.[0] || accessionRaw.split('-')[0] || '';
      const accessionClean = accessionRaw.replace(/-/g, '');

      // Direct link to the filing index page — always works
      const link = cik && accessionClean
        ? `https://www.sec.gov/Archives/edgar/data/${parseInt(cik)}/` +
          `${accessionClean}/${accessionRaw}-index.htm`
        : `https://efts.sec.gov/LATEST/search-index?q=${encodeURIComponent(ticker || company)}&forms=8-K`;

      return {
        ticker,
        company,
        date:  src.file_date ?? '',
        title: company
          ? `${company} — 8-K FDA Filing`
          : 'SEC 8-K FDA Filing',
        link,
      };
    }).filter(i => i.company || i.ticker);

    return ok({ ok: true, count: items.length, items, ts: new Date().toISOString() });
  } catch(e) { return err(e.message); }
}

// ─────────────────────────────────────────────────────────────────────────────
// /videos — Ross Cameron YouTube RSS
// ─────────────────────────────────────────────────────────────────────────────
async function handleVideos() {
  try {
    const res = await fetch('https://www.youtube.com/feeds/videos.xml?channel_id=UCBayuhgYpKNbhJxfExYkPfA', { headers: { 'User-Agent': 'Mozilla/5.0' } });
    if (!res.ok) throw new Error(`YouTube fout ${res.status}`);
    const xml = await res.text();
    const videos = [];
    const entryRe = /<entry>([\s\S]*?)<\/entry>/gi;
    let m;
    while ((m = entryRe.exec(xml)) !== null) {
      const c = m[1];
      const videoId = (c.match(/<yt:videoId>([^<]+)<\/yt:videoId>/) || [])[1] ?? '';
      const title   = (c.match(/<title>([^<]+)<\/title>/)            || [])[1] ?? '';
      const pubDate = (c.match(/<published>([^<]+)<\/published>/)    || [])[1] ?? '';
      if (!videoId) continue;
      videos.push({ videoId, title: title.replace(/&amp;/g,'&').replace(/&#39;/g,"'").trim(), pubDate, thumb: `https://i.ytimg.com/vi/${videoId}/mqdefault.jpg`, url: `https://www.youtube.com/watch?v=${videoId}` });
      if (videos.length >= 6) break;
    }
    return ok({ ok: true, count: videos.length, videos, ts: new Date().toISOString() });
  } catch(e) { return err(e.message); }
}

// ─────────────────────────────────────────────────────────────────────────────
// Yahoo Finance RSS — per ticker, very relevant
// ─────────────────────────────────────────────────────────────────────────────
async function fetchYahooRSS(ticker) {
  try {
    const url = `https://feeds.finance.yahoo.com/rss/2.0/headline?s=${ticker}&region=US&lang=en-US`;
    const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
    if (!res.ok) return [];
    const xml  = await res.text();
    const strip = s => s.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g,'$1').replace(/<[^>]+>/g,'').replace(/&amp;/g,'&').replace(/&lt;/g,'<').replace(/&gt;/g,'>').replace(/&quot;/g,'"').trim();
    const items = [];
    const itemRe = /<item>([\s\S]*?)<\/item>/gi;
    let m;
    while ((m = itemRe.exec(xml)) !== null) {
      const c       = m[1];
      const title   = strip(c.match(/<title>([\s\S]*?)<\/title>/i)?.[1] ?? '');
      const link    = strip(c.match(/<link>([\s\S]*?)<\/link>/i)?.[1] ?? '');
      const pubDate = strip(c.match(/<pubDate>([\s\S]*?)<\/pubDate>/i)?.[1] ?? '');
      if (!title || title.length < 5) continue;
      items.push({ title, link, source: 'Yahoo Finance', pubDate, tickers: [ticker] });
    }
    return items.slice(0, 5);
  } catch { return []; }
}

// ─────────────────────────────────────────────────────────────────────────────
// SEC EDGAR 8-K RSS — official filings (FDA, contracts, earnings, mergers)
// Best source for real catalyst news — always primary, never aggregated
// ─────────────────────────────────────────────────────────────────────────────
async function fetchSECFilings(ticker) {
  try {
    const url = `https://efts.sec.gov/LATEST/search-index?q=%22${ticker}%22&dateRange=custom&startdt=${getDateDaysAgo(3)}&enddt=${getTodayDate()}&forms=8-K`;
    const res = await fetch(url, { headers: { 'User-Agent': 'GapScanner contact@goldminey.com', 'Accept': 'application/json' } });
    if (!res.ok) return [];
    const data = await res.json();
    const hits  = data.hits?.hits ?? [];
    return hits.slice(0, 3).map(h => {
      const src          = h._source || {};
      const displayNames = src.display_names || [];
      const firstName    = displayNames[0] || '';
      const company      = firstName.split('(')[0].trim();
      const accessionRaw = (h._id || '').replace(/\//g, '-');
      const cik          = src.ciks?.[0] || accessionRaw.split('-')[0] || '';
      const accessionClean = accessionRaw.replace(/-/g, '');
      const link = cik && accessionClean
        ? `https://www.sec.gov/Archives/edgar/data/${parseInt(cik)}/${accessionClean}/${accessionRaw}-index.htm`
        : `https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&company=${ticker}&type=8-K&dateb=&owner=include&count=5`;
      return {
        title:   company ? `8-K Filing: ${company}` : `SEC 8-K: ${ticker}`,
        link,
        source:  'SEC EDGAR',
        pubDate: src.file_date ?? null,
        tickers: [ticker],
      };
    });
  } catch { return []; }
}

function getTodayDate() {
  return new Date().toISOString().slice(0, 10);
}

function getDateDaysAgo(days) {
  const d = new Date();
  d.setDate(d.getDate() - days);
  return d.toISOString().slice(0, 10);
}

// ─────────────────────────────────────────────────────────────────────────────
// Google News RSS per ticker — no API key, always free
// Returns all news from Google News about a specific stock
// ─────────────────────────────────────────────────────────────────────────────
async function fetchGoogleNews(ticker, companyName) {
  try {
    // Search for ticker AND company name for best results
    const query   = companyName
      ? `${ticker} stock OR "${companyName.split(' ').slice(0,3).join(' ')}"`
      : `${ticker} stock`;
    const encoded = encodeURIComponent(query);
    const url     = `https://news.google.com/rss/search?q=${encoded}&hl=en-US&gl=US&ceid=US:en`;
    const res     = await fetch(url, {
      headers: { 'User-Agent': 'Mozilla/5.0 (compatible; GapScanner/1.0)' }
    });
    if (!res.ok) return [];
    const xml   = await res.text();
    const strip = s => s
      .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
      .replace(/<[^>]+>/g, '')
      .replace(/&amp;/g,'&').replace(/&lt;/g,'<').replace(/&gt;/g,'>')
      .replace(/&quot;/g,'"').replace(/&#39;/g,"'").trim();

    const items  = [];
    const itemRe = /<item>([\s\S]*?)<\/item>/gi;
    let m;
    while ((m = itemRe.exec(xml)) !== null) {
      const c       = m[1];
      const title   = strip(c.match(/<title>([\s\S]*?)<\/title>/i)?.[1] ?? '');
      const link    = strip(c.match(/<link>([\s\S]*?)<\/link>/i)?.[1] ?? '');
      const pubDate = strip(c.match(/<pubDate>([\s\S]*?)<\/pubDate>/i)?.[1] ?? '');
      // Extract source name from <source> tag
      const source  = strip(c.match(/<source[^>]*>([\s\S]*?)<\/source>/i)?.[1] ?? 'Google News');
      if (!title || title.length < 5) continue;
      items.push({ title, link, source, pubDate, tickers: [ticker] });
    }
    return items.slice(0, 5);
  } catch { return []; }
}

// ─────────────────────────────────────────────────────────────────────────────
// RSS feeds
// ─────────────────────────────────────────────────────────────────────────────
async function fetchAllRSS() {
  const results = await Promise.allSettled(RSS_FEEDS.map(f => fetchRSS(f.url, f.source)));
  const seen = new Set(), all = [];
  for (const r of results) {
    if (r.status !== 'fulfilled') continue;
    for (const item of r.value) {
      const k = item.title?.slice(0,60);
      if (!k || seen.has(k)) continue;
      seen.add(k); all.push(item);
    }
  }
  return all.sort((a,b) => new Date(b.pubDate||0)-new Date(a.pubDate||0));
}

async function fetchRSS(url, source) {
  try {
    const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0', 'Accept': 'application/rss+xml,*/*' } });
    if (!res.ok) return [];
    return parseRSS(await res.text(), source);
  } catch { return []; }
}

function parseRSS(xml, source) {
  const items = [];
  const strip = s => s.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g,'$1').replace(/<[^>]+>/g,'').replace(/&amp;/g,'&').replace(/&lt;/g,'<').replace(/&gt;/g,'>').replace(/&quot;/g,'"').replace(/&#39;/g,"'").trim();
  const SKIP  = new Set(['FDA','CEO','CFO','IPO','ETF','NYSE','SEC','USD','GDP','AI','US','UK','EU','Q1','Q2','Q3','Q4','AM','PM','EST','ET','LLC','INC','LTD','THE','AND','FOR','NEW','NOT','ALL','TOP']);
  const itemRe = /<item[^>]*>([\s\S]*?)<\/item>/gi;
  let m;
  while ((m = itemRe.exec(xml)) !== null) {
    const c = m[1];
    const title   = strip(c.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]??'');
    const link    = strip(c.match(/<link[^>]*>([\s\S]*?)<\/link>/i)?.[1]??'');
    const pubDate = strip(c.match(/<pubDate[^>]*>([\s\S]*?)<\/pubDate>/i)?.[1]??'');
    if (!title || title.length < 5) continue;
    const tickers = [...new Set([
      ...[...(title).matchAll(/\(\$?([A-Z]{1,6})\)/g)].map(t=>t[1]),
      ...[...(title).matchAll(/\b([A-Z]{2,5})\b/g)].map(t=>t[1]).filter(t=>!SKIP.has(t)),
    ])];
    items.push({ title, link, pubDate, source, tickers });
  }
  return items;
}

function calcScore(gapPct, relVol, price, hasNews = false) {
  let s = Math.min(40, (gapPct / 100) * 40);
  if (relVol != null) s += Math.min(25, ((relVol-1)/9)*25);
  if (price >= 1  && price <= 20)  s += 10;
  else if (price > 20 && price <= 100) s += 5;
  if (hasNews) s += 20;
  return Math.round(Math.min(100, s));
}

// Score a news item by source quality, catalyst keywords and recency
function newsScore(n) {
  let s = 0;
  const src   = (n.source  || '').toLowerCase();
  const title = (n.title   || n.headline || '').toLowerCase();
  if (src.includes('sec.gov') || src === 'sec edgar')          s += 50;
  if (src.includes('businesswire'))                            s += 30;
  if (src.includes('globenewswire'))                           s += 25;
  if (src.includes('prnewswire') || src.includes('pr newswire')) s += 25;
  if (src.includes('accesswire'))                              s += 25;
  if (src.includes('yahoo'))                                   s += 20;
  if (src.includes('benzinga'))                                s += 15;
  if (src.includes('reuters'))                                 s += 20;
  if (src.includes('bloomberg'))                               s += 20;
  if (src.includes('marketwatch'))                             s += 12;
  if (src.includes('cnbc'))                                    s += 12;
  if (src === 'google news' || src.includes('google'))         s += 8;
  const catalysts = ['fda','approval','approved','clearance','clears',
    'earnings','revenue','beat','guidance','merger','acquisition','acquires',
    'contract','award','partnership','deal','ipo','offering','uplisting',
    'clinical','trial','phase','data','results','grant','license',
    'breakthrough','record','expands','launches','announces'];
  if (catalysts.some(w => title.includes(w))) s += 20;
  const ts = n.datetime || (n.pubDate ? new Date(n.pubDate).getTime()/1000 : 0);
  const ageHrs = (Date.now()/1000 - ts) / 3600;
  if (ageHrs < 1)       s += 15;
  else if (ageHrs < 4)  s += 8;
  else if (ageHrs < 12) s += 3;
  return s;
}

function ok(data)       { return new Response(JSON.stringify(data), { headers: { ...CORS, 'Content-Type': 'application/json' } }); }
function err(msg, status=500) { return new Response(JSON.stringify({ ok: false, error: msg }), { status, headers: { ...CORS, 'Content-Type': 'application/json' } }); }