Web Scraper Tool

The previous example was quite simple. In order to get more complex functionality we can utilize api calls to gather useful information from the web. Below is a simple example that allows an agent to scrape a website URL and return the context back to the conversation.


import { tool } from "ai";
import * as z from "zod";
import FirecrawlApp, { ScrapeResponse } from "@mendable/firecrawl-js";
import { logger } from "@agentpress/lib/utils/logger";
 
if (!process.env.FIRECRAWL_API_KEY) {
  logger.warn("FIRECRAWL_API_KEY is not defined");
}
 
const app = new FirecrawlApp({
  apiKey: process.env.FIRECRAWL_API_KEY as string,
});
 
export const scrape = tool({
  description: "Scrape a website for information.",
  parameters: z.object({
    url: z.string().describe("The URL of the website"),
  }),
  execute: async ({ url }) => {
    if (!process.env.FIRECRAWL_API_KEY) {
      return {
        error: "FIRECRAWL_API_KEY is not defined",
      };
    }
    try {
      const scrapeResult = (await app.scrapeUrl(url, {
        formats: ["markdown"],
      })) as ScrapeResponse;
      if (!scrapeResult.success) {
        return {
          error: scrapeResult.error,
        };
      }
      console.log(`Scrape url response: ${JSON.stringify(scrapeResult)}`);
      return scrapeResult;
    } catch (error) {
      logger.error(`Scrape url error: ${error}`);
      return {
        error: `Unable to scrape website. This is likely due to an invalid URL or network issue. Double check the URL and try again.`,
      };
    }
  },
});