From 810266680af08107f8e926497318ae4feefd06bc Mon Sep 17 00:00:00 2001 From: mpl Date: Tue, 30 Jun 2020 01:00:11 +0200 Subject: [PATCH] Cope when item in .lastDone does not exist (anymore) Also update chromedp to 0.5.4, so we can benefit from conveniences such as RunResponse (which is relied upon for this fix). Consequently, that means a recent enough Chrome version (v83) is needed as per this commit. See https://github.com/chromedp/chromedp/issues/627 Fixes #17 --- go.mod | 4 ++-- go.sum | 19 +++++++++++++++++++ main.go | 37 ++++++++++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 83bad28..040ff8a 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/perkeep/gphotos-cdp go 1.12 require ( - github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d - github.com/chromedp/chromedp v0.4.0 + github.com/chromedp/cdproto v0.0.0-20200608134039-8a80cdaf865c + github.com/chromedp/chromedp v0.5.4-0.20200624114048-353306f986a8 ) diff --git a/go.sum b/go.sum index 2da1a9c..645a5cd 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,35 @@ github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d h1:00kLGv5nKzpFchNhGDXDRbKtYx/WoT983Ka2t8/pzRE= github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= +github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac h1:T7V5BXqnYd55Hj/g5uhDYumg9Fp3rMTS6bykYtTIFX4= +github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g= +github.com/chromedp/cdproto v0.0.0-20200209033844-7e00b02ea7d2/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g= +github.com/chromedp/cdproto v0.0.0-20200608134039-8a80cdaf865c h1:qM1xzKK8kc93zKPkxK4iqtjksqDDrU6g9wGnr30jyLo= +github.com/chromedp/cdproto v0.0.0-20200608134039-8a80cdaf865c/go.mod h1:E6LPWRdIJc11h/di5p0rwvRmUYbhGpBEH7ZbPfzDIOE= github.com/chromedp/chromedp v0.4.0 h1:0AJC5ejETuh/6n7Tcsw4u4G0eKZkI9aVRwckWaImLUE= github.com/chromedp/chromedp v0.4.0/go.mod h1:DC3QUn4mJ24dwjcaGQLoZrhm4X/uPHZ6spDbS2uFhm4= +github.com/chromedp/chromedp v0.5.3 h1:F9LafxmYpsQhWQBdCs+6Sret1zzeeFyHS5LkRF//Ffg= +github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aaLxYgYj28w= +github.com/chromedp/chromedp v0.5.4-0.20200624114048-353306f986a8 h1:JU/73ZWIwh2HUPEt1pFKoVOhT8hMS6Q9WQ382v0WUjI= +github.com/chromedp/chromedp v0.5.4-0.20200624114048-353306f986a8/go.mod h1:fIICWzcY8T8pfGlPyweJ5MNnBi2dGZgoRn/t4BOu7gA= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gobwas/ws v1.0.3 h1:ZOigqf7iBxkA4jdQ3am7ATzdlOFp9YzA6NmuvEEZc9g= +github.com/gobwas/ws v1.0.3/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307 h1:vl4eIlySbjertFaNwiMjXsGrFVK25aOWLq7n+3gh2ls= github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ= +github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs= +github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/main.go b/main.go index df55c6d..a478ad3 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( "fmt" "io/ioutil" "log" + "net/http" "os" "os/exec" "path/filepath" @@ -34,6 +35,7 @@ import ( "sync" "time" + "github.com/chromedp/cdproto/browser" "github.com/chromedp/cdproto/input" "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" @@ -190,6 +192,9 @@ func (s *Session) cleanDlDir() error { if v.IsDir() { continue } + if v.Name() == ".lastdone" { + continue + } if err := os.Remove(filepath.Join(s.dlDir, v.Name())); err != nil { return err } @@ -201,7 +206,7 @@ func (s *Session) cleanDlDir() error { // authenticated (or for 2 minutes to have elapsed). func (s *Session) login(ctx context.Context) error { return chromedp.Run(ctx, - page.SetDownloadBehavior(page.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir), + browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir), chromedp.ActionFunc(func(ctx context.Context) error { if *verboseFlag { log.Printf("pre-navigate") @@ -255,14 +260,40 @@ func (s *Session) firstNav(ctx context.Context) error { } if *startFlag != "" { + // TODO(mpl): use RunResponse chromedp.Navigate(*startFlag).Do(ctx) chromedp.WaitReady("body", chromedp.ByQuery).Do(ctx) return nil } if s.lastDone != "" { - chromedp.Navigate(s.lastDone).Do(ctx) + resp, err := chromedp.RunResponse(ctx, chromedp.Navigate(s.lastDone)) + if err != nil { + return err + } + if resp.Status == http.StatusOK { + chromedp.WaitReady("body", chromedp.ByQuery).Do(ctx) + return nil + } + lastDoneFile := filepath.Join(s.dlDir, ".lastdone") + log.Printf("%s does not seem to exist anymore. Removing %s.", s.lastDone, lastDoneFile) + s.lastDone = "" + if err := os.Remove(lastDoneFile); err != nil { + if os.IsNotExist(err) { + log.Fatal("Failed to remove .lastdone file because it was already gone.") + } + return err + } + + // restart from scratch + resp, err = chromedp.RunResponse(ctx, chromedp.Navigate("https://photos.google.com/")) + if err != nil { + return err + } + code := resp.Status + if code != http.StatusOK { + return fmt.Errorf("unexpected %d code when restarting to https://photos.google.com/", code) + } chromedp.WaitReady("body", chromedp.ByQuery).Do(ctx) - return nil } if err := navToEnd(ctx); err != nil {