gddo-admin/crawl.go

// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/garyburd/redigo/redis"
	"github.com/golang/gddo/database"
)

var crawlCommand = &command{
	name:  "crawl",
	run:   crawl,
	usage: "crawl [new]",
}

func crawl(c *command) {
	if len(c.flag.Args()) > 1 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
	if err != nil {
		log.Fatal(err)
	}

	if len(c.flag.Args()) == 1 {
		p, err := ioutil.ReadFile(c.flag.Args()[0])
		if err != nil {
			log.Fatal(err)
		}
		for _, p := range strings.Fields(string(p)) {
			db.AddNewCrawl(p)
		}
	}

	conn := db.Pool.Get()
	defer conn.Close()
	paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("NEW")
	for _, path := range paths {
		fmt.Println(path)
	}

	paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("BAD")
	for _, path := range paths {
		fmt.Println(path)
	}
}