{"id":1619,"date":"2012-04-03T14:25:30","date_gmt":"2012-04-03T05:25:30","guid":{"rendered":"http:\/\/peta.okechan.net\/blog\/?p=1619"},"modified":"2012-04-03T17:50:28","modified_gmt":"2012-04-03T08:50:28","slug":"a-tour-of-go%e3%81%ae69","status":"publish","type":"post","link":"https:\/\/peta.okechan.net\/blog\/archives\/1619","title":{"rendered":"A Tour of Go\u306e69"},"content":{"rendered":"

A Tour of Go\u306e69 Exercise: Web Crawler<\/a>\u3092\u3084\u3063\u3066\u307f\u307e\u3057\u305f\u3002
\n68\u3092\u3084\u3063\u305f\u76f4\u5f8c\u304b\u3089\u8272\u3005\u8003\u3048\u305f\u308a\u8a66\u3057\u3066\u307f\u305f\u308a\u3057\u3066\u305f\u3093\u3067\u3059\u304c\u3001\u3046\u307e\u304f\u884c\u304b\u305a\u3001\u4e381\u65e5\u7a0b\u5ea6\u9593\u3092\u958b\u3051\u3066\u304b\u3089\u6700\u521d\u304b\u3089\u8003\u3048\u306a\u304a\u3057\u3066\u307f\u305f\u3089\u3042\u3063\u3055\u308a\u3044\u304d\u307e\u3057\u305f\u3002<\/p>\n

\u52b9\u679c\u3092\u5b9f\u611f\u3057\u3084\u3059\u304f\u3059\u308b\u305f\u3081\u3001Fetch\u3067time.Sleep\u3059\u308b\u3088\u3046\u306b\u3057\u3066\u307e\u3059\u304c\u3001\u30aa\u30f3\u30e9\u30a4\u30f3\u306eA Tour of Go\u3067\u306ftime.Sleep\u306f\u4f7f\u3048\u306a\u3044\u307f\u305f\u3044\u306a\u306e\u3067\u3001\u30ed\u30fc\u30ab\u30eb\u306e\u3082\u306e\u3067\u8a66\u3057\u3066\u307e\u3059\u3002
\n\u3068\u3044\u3046\u304b\u3001\u30aa\u30f3\u30e9\u30a4\u30f3\u306eA Tour of Go\u306f\u5236\u9650\u304c\u3042\u308b\u3057\u3001\u30ed\u30fc\u30ab\u30eb\u306eA Tour of Go\u306f\u7121\u9650\u30eb\u30fc\u30d7\u306b\u9665\u3063\u305f\u308a\u3057\u305f\u3068\u304d\u306b\u3084\u308a\u76f4\u3057\u304c\u9762\u5012\u3060\u3057\u3067\u3001\u666e\u901a\u306b\u30c6\u30ad\u30b9\u30c8\u30a8\u30c7\u30a3\u30bf\u3067\u30bd\u30fc\u30b9\u66f8\u3044\u3066\u3001\u30b3\u30de\u30f3\u30c9\u30e9\u30a4\u30f3\u3067\u30b3\u30f3\u30d1\u30a4\u30eb\u30fb\u5b9f\u884c\u3057\u305f\u307b\u3046\u304c\u3001\u3061\u3087\u3063\u3068\u5909\u66f4\u3057\u3066\u306f\u8a66\u3057\u3092\u7e70\u308a\u8fd4\u3059\u3068\u304d\u306f\u4fbf\u5229\u3067\u3059\u3002<\/p>\n

\u3053\u3046\u3044\u3046\u66f8\u304d\u65b9\u3067\u3044\u3044\u3093\u3067\u3059\u304b\u306d\u3002
\n\u30b5\u30f3\u30d7\u30eb\u30c7\u30fc\u30bf\u304c\u3057\u3087\u307c\u3044\u304b\u3089\u3044\u3044\u3067\u3059\u3051\u3069\u3001\u5b9f\u969b\u306f\u540c\u6642\u306b\u5b9f\u884c\u3059\u308b\u30b9\u30ec\u30c3\u30c9\u6570\u306f\u5236\u9650\u3057\u305f\u307b\u3046\u304c\u3044\u3044\u306f\u305a\u3002<\/p>\n

\u8ffd\u8a18<\/h3>\n

Next\u69cb\u9020\u4f53\u306fCrawlCommand\u307f\u305f\u3044\u306a\u540d\u524d\u306b\u3059\u3079\u304d\u3060\u3063\u305f\u3002
\nchan CrawlResult\u3058\u3083\u306a\u304f\u3066chan *CrawlResult\u306a\u30c1\u30e3\u30f3\u30cd\u30eb\u3092\u4f7f\u3046\u3068\u3001\u7d50\u679c\u306e\u6709\u7121\u3092nil\u3068\u306e\u6bd4\u8f03\u3067\u5224\u5b9a\u3067\u304d\u308b\u3057\u3001\u52b9\u7387\u4e0a\u3082\u6709\u5229\u304b\u3082\u3057\u308c\u306a\u3044\u3002<\/p>\n

package main\r\n\r\nimport (\r\n    "fmt"\r\n    "time"\r\n)\r\n\r\ntype Fetcher interface {\r\n    \/\/ Fetch returns the body of URL and\r\n    \/\/ a slice of URLs found on that page.\r\n    Fetch(url string) (body string, urls []string, err error)\r\n}\r\n\r\n\/\/ Crawl uses fetcher to recursively crawl\r\n\/\/ pages starting with url, to a maximum of depth.\r\nfunc Crawl(url string, depth int, fetcher Fetcher, ch chan CrawlResult) {\r\n    \/\/ This implementation doesn't do either:\r\n    if depth <= 0 {\r\n        ch <- CrawlResult{}\r\n        return\r\n    }\r\n    body, urls, err := fetcher.Fetch(url)\r\n    if err != nil {\r\n        ch <- CrawlResult{Err: err}\r\n        return\r\n    }\r\n    fmt.Printf("found: %s %q\\n", url, body)\r\n    ch <- CrawlResult{URLs: urls, Crawled: url, Depth: depth}\r\n    return\r\n}\r\n\r\ntype Next struct {\r\n    URL string\r\n    Depth int\r\n}\r\n\r\ntype CrawlResult struct {\r\n    URLs []string\r\n    Crawled string\r\n    Depth int\r\n    Err error\r\n}\r\n\r\nfunc main() {\r\n    ch := make(chan CrawlResult)\r\n    nexts := []Next{Next{"http:\/\/golang.org\/", 4}}\r\n    crawleds := make(map[string]int)\r\n    \r\n    for len(nexts) > 0 {\r\n        goroutines := 0\r\n        for _, n := range nexts {\r\n            if crawleds[n.URL] > 0 {continue}\r\n\r\n            go Crawl(n.URL, n.Depth, fetcher, ch)\r\n            goroutines++\r\n            fmt.Printf("goroutine %d start\\n", goroutines)\r\n        }\r\n\r\n        nexts = make([]Next, 0)\r\n        for i := 0; i < goroutines; i++ {\r\n            ret := <-ch\r\n            if ret.Err != nil {fmt.Println(ret.Err)}\r\n            if ret.Crawled == "" {continue}\r\n            \r\n            crawleds[ret.Crawled]++\r\n            for _, url := range ret.URLs {\r\n                nexts = append(nexts, Next{url, ret.Depth-1})\r\n            }\r\n        }\r\n    }\r\n}\r\n\r\n\r\n\/\/ fakeFetcher is Fetcher that returns canned results.\r\ntype fakeFetcher map[string]*fakeResult\r\n\r\ntype fakeResult struct {\r\n    body string\r\n    urls     []string\r\n}\r\n\r\nfunc (f *fakeFetcher) Fetch(url string) (string, []string, error) {\r\n    time.Sleep(1000 * time.Millisecond)\r\n    if res, ok := (*f)[url]; ok {\r\n        return res.body, res.urls, nil\r\n    }\r\n    return "", nil, fmt.Errorf("not found: %s", url)\r\n}\r\n\r\n\/\/ fetcher is a populated fakeFetcher.\r\nvar fetcher = &fakeFetcher{\r\n    "http:\/\/golang.org\/": &fakeResult{\r\n        "The Go Programming Language",\r\n        []string{\r\n            "http:\/\/golang.org\/pkg\/",\r\n            "http:\/\/golang.org\/cmd\/",\r\n        },\r\n    },\r\n    "http:\/\/golang.org\/pkg\/": &fakeResult{\r\n        "Packages",\r\n        []string{\r\n            "http:\/\/golang.org\/",\r\n            "http:\/\/golang.org\/cmd\/",\r\n            "http:\/\/golang.org\/pkg\/fmt\/",\r\n            "http:\/\/golang.org\/pkg\/os\/",\r\n        },\r\n    },\r\n    "http:\/\/golang.org\/pkg\/fmt\/": &fakeResult{\r\n        "Package fmt",\r\n        []string{\r\n            "http:\/\/golang.org\/",\r\n            "http:\/\/golang.org\/pkg\/",\r\n        },\r\n    },\r\n    "http:\/\/golang.org\/pkg\/os\/": &fakeResult{\r\n        "Package os",\r\n        []string{\r\n            "http:\/\/golang.org\/",\r\n            "http:\/\/golang.org\/pkg\/",\r\n        },\r\n    },\r\n}<\/pre>\n","protected":false},"excerpt":{"rendered":"
A Tour of Go\u306e69 Exercise: Web Crawler<\/a>\u3092\u3084\u3063\u3066\u307f\u307e\u3057\u305f\u3002
\n68\u3092\u3084\u3063\u305f\u76f4\u5f8c\u304b\u3089\u8272\u3005\u8003\u3048\u305f\u308a\u8a66\u3057\u3066\u307f\u305f\u308a\u3057\u3066\u305f\u3093\u3067\u3059\u304c\u3001\u3046\u307e\u304f\u884c\u304b\u305a\u3001\u4e381\u65e5\u7a0b\u5ea6\u9593\u3092\u958b\u3051\u3066\u304b\u3089\u6700\u521d\u304b\u3089\u8003\u3048\u306a\u304a\u3057\u3066\u307f\u305f\u3089\u3042\u3063\u3055\u308a\u3044\u304d\u307e\u3057\u305f\u3002<\/p>\n
\u52b9\u679c\u3092\u5b9f\u611f\u3057\u3084\u3059\u304f\u3059\u308b\u305f\u3081\u3001Fetch\u3067time.Sleep\u3059\u308b\u3088\u3046\u306b\u3057\u3066\u307e\u3059\u304c\u3001\u30aa\u30f3\u30e9\u30a4\u30f3\u306eA Tour of Go\u3067\u306ftime.Sleep\u306f\u4f7f\u3048\u306a\u3044\u307f\u305f\u3044\u306a\u306e\u3067\u3001\u30ed\u30fc\u30ab\u30eb\u306e\u3082\u306e\u3067\u8a66\u3057\u3066\u307e\u3059\u3002
\n\u3068\u3044\u3046\u304b\u3001\u30aa\u30f3\u30e9\u30a4\u30f3\u306eA Tour of Go\u306f\u5236\u9650\u304c\u3042\u308b\u3057\u3001\u30ed\u30fc\u30ab\u30eb\u306eA Tour of Go\u306f\u7121\u9650\u30eb\u30fc\u30d7\u306b\u9665\u3063\u305f\u308a\u3057\u305f\u3068\u304d\u306b\u3084\u308a\u76f4\u3057\u304c\u9762\u5012\u3060\u3057\u3067\u3001\u666e\u901a\u306b\u30c6\u30ad\u30b9\u30c8\u30a8\u30c7\u30a3\u30bf\u3067\u30bd\u30fc\u30b9\u66f8\u3044\u3066\u3001\u30b3\u30de\u30f3\u30c9\u30e9\u30a4\u30f3\u3067\u30b3\u30f3\u30d1\u30a4\u30eb\u30fb\u5b9f\u884c\u3057\u305f\u307b\u3046\u304c\u3001\u3061\u3087\u3063\u3068\u5909\u66f4\u3057\u3066\u306f\u8a66\u3057\u3092\u7e70\u308a\u8fd4\u3059\u3068\u304d\u306f\u4fbf\u5229\u3067\u3059\u3002<\/p>\n
\u3053\u3046\u3044\u3046\u66f8\u304d\u65b9\u3067\u3044\u3044\u3093\u3067\u3059\u304b\u306d\u3002
\n\u30b5\u30f3\u30d7\u30eb\u30c7\u30fc\u30bf\u304c\u3057\u3087\u307c\u3044\u304b\u3089\u3044\u3044\u3067\u3059\u3051\u3069\u3001\u5b9f\u969b\u306f\u540c\u6642\u306b\u5b9f\u884c\u3059\u308b\u30b9\u30ec\u30c3\u30c9\u6570\u306f\u5236\u9650\u3057\u305f\u307b\u3046\u304c\u3044\u3044\u306f\u305a\u3002<\/p>\n
\u8ffd\u8a18<\/h3>\n
Next\u69cb\u9020\u4f53\u306fCrawlCommand\u307f\u305f\u3044\u306a\u540d\u524d\u306b\u3059\u3079\u304d\u3060\u3063\u305f\u3002
\nchan CrawlResult\u3058\u3083\u306a\u304f\u3066chan *CrawlResult\u306a\u30c1\u30e3\u30f3\u30cd\u30eb\u3092\u4f7f\u3046\u3068\u3001\u7d50\u679c\u306e\u6709\u7121\u3092nil\u3068\u306e\u6bd4\u8f03\u3067\u5224\u5b9a\u3067\u304d\u308b\u3057\u3001\u52b9\u7387\u4e0a\u3082\u6709\u5229\u304b\u3082\u3057\u308c\u306a\u3044\u3002<\/p>\n
package main\r\n\r\nimport (\r\n    "fmt"\r\n    "time"\r\n)\r\n\r\ntype Fetcher interface {\r\n    \/\/ Fetch returns the body of URL and\r\n    \/\/ a slice of URLs found on that page.\r\n    Fetch(url string) (body string, urls []string, err error)\r\n}\r\n\r\n\/\/ Crawl uses fetcher to recursively crawl\r\n\/\/ pages starting with url, to a maximum of depth.\r\nfunc Crawl(url string, depth int, fetcher Fetcher, ch chan CrawlResult) {\r\n    \/\/ This implementation doesn't do either:\r\n    if depth <= 0 {\r\n        ch <- CrawlResult{}\r\n        return\r\n    }\r\n    body, urls, err := fetcher.Fetch(url)\r\n    if err != nil {\r\n        ch <- CrawlResult{Err: err}\r\n        return\r\n    }\r\n    fmt.Printf("found: %s %q\\n", url, body)\r\n    ch <- CrawlResult{URLs: urls, Crawled: url, Depth: depth}\r\n    return\r\n}\r\n\r\ntype Next struct {\r\n    URL string\r\n    Depth int\r\n}\r\n\r\ntype CrawlResult struct {\r\n    URLs []string\r\n    Crawled string\r\n    Depth int\r\n    Err error\r\n}\r\n\r\nfunc main() {\r\n    ch := make(chan CrawlResult)\r\n    nexts := []Next{Next{"http:\/\/golang.org\/", 4}}\r\n    crawleds := make(map[string]int)\r\n    \r\n    for len(nexts) > 0 {\r\n        goroutines := 0\r\n        for _, n := range nexts {\r\n            if crawleds[n.URL] > 0 {continue}\r\n\r\n            go Crawl(n.URL, n.Depth, fetcher, ch)\r\n            goroutines++\r\n            fmt.Printf("goroutine %d start\\n", goroutines)\r\n        }\r\n\r\n        nexts = make([]Next, 0)\r\n        for i := 0; i < goroutines; i++ {\r\n            ret := <-ch\r\n            if ret.Err != nil {fmt.Println(ret.Err)}\r\n            if ret.Crawled == "" {continue}\r\n            \r\n            crawleds[ret.Crawled]++\r\n            for _, url := range ret.URLs {\r\n                nexts = append(nexts, Next{url, ret.Depth-1})\r\n            }\r\n        }\r\n    }\r\n}\r\n\r\n\r\n\/\/ fakeFetcher is Fetcher that returns canned results.\r\ntype fakeFetcher map[string]*fakeResult\r\n\r\ntype fakeResult struct {\r\n    body string\r\n    urls     []string\r\n}\r\n\r\nfunc (f *fakeFetcher) Fetch(url string) (string, []string, error) {\r\n    time.Sleep(1000 * time.Millisecond)\r\n    if res, ok := (*f)[url]; ok {\r\n        return res.body, res.urls, nil\r\n    }\r\n    return "", nil, fmt.Errorf("not found: %s", url)\r\n}\r\n\r\n\/\/ fetcher is a populated fakeFetcher.\r\nvar fetcher = &fakeFetcher{\r\n    "http:\/\/golang.org\/": &fakeResult{\r\n        "The Go Programming Language",\r\n        []string{\r\n            "http:\/\/golang.org\/pkg\/",\r\n            "http:\/\/golang.org\/cmd\/",\r\n        },\r\n    },\r\n    "http:\/\/golang.org\/pkg\/": &fakeResult{\r\n        "Packages",\r\n        []string{\r\n            "http:\/\/golang.org\/",\r\n            "http:\/\/golang.org\/cmd\/",\r\n            "http:\/\/golang.org\/pkg\/fmt\/",\r\n            "http:\/\/golang.org\/pkg\/os\/",\r\n        },\r\n    },\r\n    "http:\/\/golang.org\/pkg\/fmt\/": &fakeResult{\r\n        "Package fmt",\r\n        []string{\r\n            "http:\/\/golang.org\/",\r\n            "http:\/\/golang.org\/pkg\/",\r\n        },\r\n    },\r\n    "http:\/\/golang.org\/pkg\/os\/": &fakeResult{\r\n        "Package os",\r\n        []string{\r\n            "http:\/\/golang.org\/",\r\n            "http:\/\/golang.org\/pkg\/",\r\n        },\r\n    },\r\n}<\/pre>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[32],"tags":[410],"class_list":["post-1619","post","type-post","status-publish","format-standard","hentry","category-tech","tag-go"],"_links":{"self":[{"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/posts\/1619","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/comments?post=1619"}],"version-history":[{"count":0,"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/posts\/1619\/revisions"}],"wp:attachment":[{"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/media?parent=1619"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/categories?post=1619"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/peta.okechan.net\/blog\/wp-json\/wp\/v2\/tags?post=1619"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}