{
  "id": "2004-september-october-scraping-away-for-useful-data",
  "slug": "2004-september-october-scraping-away-for-useful-data",
  "url": "https://dwillis.github.io/uplink/article/2004-september-october-scraping-away-for-useful-data.html",
  "year": 2004,
  "month": "September • October",
  "monthNum": 9,
  "headline": "Scraping away for useful data",
  "kicker": "INSIDE NICAR",
  "author_name": "Derek Willis and Jeff Porter",
  "author_title": "",
  "affiliation": "The Center for Public Integrity / IRE and NICAR",
  "summary": "IRE and NICAR are launching the Scraping Project to help journalists use programming scripts to extract and format data from public government websites.",
  "keywords": [
    "web scraping",
    "data collection",
    "Perl"
  ],
  "topics": [
    "Investigative Methods",
    "Technology",
    "Data & Databases"
  ],
  "technologies": [
    "Perl",
    "PHP",
    "Python",
    "spreadsheet",
    "Internet",
    "World Wide Web"
  ],
  "full_text": "Been scraping lately? Toss out the mental images of home improvement, and instead think about databases on the Internet. The rise of government Web sites provides a great opportunity for CAR-savvy newsrooms. As with many developments involving governments, the promise is somewhat hidden, but IRE and NICAR are planning a project to change at least part of that. More and more public databases are being placed on the Web, but often they are hidden behind a search form, or they require hours of repetition to save each record. And then there's the time spent separating the data from the HTML that surrounds it. The Scraping Project is an answer to these problems. With a small dose of programming (think scripting languages like Perl, PHP or Python), news organizations can \"scrape\" the information directly from a Web site and save it in a format that spreadsheet and database programs can use. Many state and federal agencies now provide data in this way, even if the underlying data can be obtained through public record requests. The advantage of scraping is that often you can get the data in a much quicker time frame. Here are some recent examples of scraping: Mark Schaver of The (Louisville) Courier-Journal wrote a script in Perl to grab information about state-registered charities from the Kentucky attorney general's Web site. In another instance, John Perry of The Oklahoman used Perl to scrape docket information off a state court Web site. Other IRE members have done – or will soon do – similar tasks, and the Scraping Project will become a central resource for either the code or the data (or both, depending on the situation). Journalists would identify data sources in their states or cities that are not available as a single download, and then either write a script themselves or ask for assistance from fellow scrapers. Depending on the nature of the data and competitive factors, the script – and perhaps the data – would be made available to other journalists through IRE's Web site. For news about the Scraping Project, keep checking the main IRE Web page (www.ire.org). Developments will also be announced on the NICAR-L listserv.",
  "provenance": {
    "extracted_by": "pre-v2",
    "extracted_at": "",
    "verified": true
  },
  "similar": [
    {
      "slug": "2004-november-scrape-away",
      "score": 0.698
    },
    {
      "slug": "2004-september-october-scaping-for-data-the-perl-way",
      "score": 0.414
    },
    {
      "slug": "2000-june-the-perl-y-gates",
      "score": 0.221
    },
    {
      "slug": "2000-julyaugust-more-on-perl",
      "score": 0.146
    },
    {
      "slug": "1996-december-from-the-nicar-library",
      "score": 0.139
    }
  ],
  "html_url": "https://dwillis.github.io/uplink/article/2004-september-october-scraping-away-for-useful-data.html",
  "json_url": "https://dwillis.github.io/uplink/article/2004-september-october-scraping-away-for-useful-data.json"
}