You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
474 lines
12 KiB
Go
474 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/chromedp/cdproto/input"
|
|
"github.com/chromedp/cdproto/page"
|
|
"github.com/chromedp/chromedp"
|
|
"github.com/chromedp/chromedp/kb"
|
|
)
|
|
|
|
var (
|
|
nItemsFlag = flag.Int("n", -1, "number of items to download. If negative, get them all.")
|
|
devFlag = flag.Bool("dev", false, "dev mode. we reuse the same session dir (/tmp/gphotos-cdp), so we don't have to auth at every run.")
|
|
dlDirFlag = flag.String("dldir", "", "where to (temporarily) write the downloads. defaults to $HOME/Downloads/gphotos-cdp.")
|
|
)
|
|
|
|
func main() {
|
|
flag.Parse()
|
|
if *nItemsFlag == 0 {
|
|
return
|
|
}
|
|
s, err := NewSession()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer s.Shutdown()
|
|
|
|
log.Printf("Session Dir: %v", s.profileDir)
|
|
|
|
if err := s.cleanDlDir(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
ctx, cancel := s.NewContext()
|
|
defer cancel()
|
|
|
|
var outerBefore string
|
|
|
|
// login phase
|
|
if err := chromedp.Run(ctx,
|
|
chromedp.ActionFunc(func(ctx context.Context) error {
|
|
log.Printf("pre-navigate")
|
|
return nil
|
|
}),
|
|
chromedp.Navigate("https://photos.google.com/"),
|
|
// chromedp.Sleep(30000*time.Millisecond),
|
|
chromedp.Sleep(5000*time.Millisecond),
|
|
chromedp.ActionFunc(func(ctx context.Context) error {
|
|
log.Printf("post-navigate")
|
|
return nil
|
|
}),
|
|
chromedp.OuterHTML("html>body", &outerBefore),
|
|
chromedp.ActionFunc(func(ctx context.Context) error {
|
|
log.Printf("Source is %d bytes", len(outerBefore))
|
|
return nil
|
|
}),
|
|
); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
navEnd := func(ctx context.Context) error {
|
|
keyEnd, ok := kb.Keys['\u0305']
|
|
if !ok {
|
|
return errors.New("no End key")
|
|
}
|
|
|
|
down := input.DispatchKeyEventParams{
|
|
Key: keyEnd.Key,
|
|
Code: keyEnd.Code,
|
|
// Some github issue says to remove NativeVirtualKeyCode, but it does not change anything.
|
|
NativeVirtualKeyCode: keyEnd.Native,
|
|
WindowsVirtualKeyCode: keyEnd.Windows,
|
|
Type: input.KeyDown,
|
|
}
|
|
if runtime.GOOS == "darwin" {
|
|
down.NativeVirtualKeyCode = 0
|
|
}
|
|
up := down
|
|
up.Type = input.KeyUp
|
|
|
|
for _, ev := range []*input.DispatchKeyEventParams{&down, &up} {
|
|
// log.Printf("Event: %+v", *ev)
|
|
if err := ev.Do(ctx); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
time.Sleep(5 * time.Second)
|
|
return nil
|
|
}
|
|
|
|
markDone := func(dldir, location string) error {
|
|
println("LOCATION: ", location)
|
|
// TODO(mpl): back up .lastdone before overwriting it, in case writing it fails.
|
|
if err := ioutil.WriteFile(filepath.Join(dldir, ".lastdone"), []byte(location), 0600); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
download := func(ctx context.Context, location string, doMarkDone bool) (string, error) {
|
|
dir := s.dlDir
|
|
keyD, ok := kb.Keys['D']
|
|
if !ok {
|
|
log.Fatal("NO D KEY")
|
|
}
|
|
|
|
down := input.DispatchKeyEventParams{
|
|
Key: keyD.Key,
|
|
Code: keyD.Code,
|
|
// Some github issue says to remove NativeVirtualKeyCode, but it does not change anything.
|
|
NativeVirtualKeyCode: keyD.Native,
|
|
WindowsVirtualKeyCode: keyD.Windows,
|
|
Type: input.KeyDown,
|
|
Modifiers: input.ModifierShift,
|
|
}
|
|
if runtime.GOOS == "darwin" {
|
|
down.NativeVirtualKeyCode = 0
|
|
}
|
|
up := down
|
|
up.Type = input.KeyUp
|
|
|
|
for _, ev := range []*input.DispatchKeyEventParams{&down, &up} {
|
|
// log.Printf("Event: %+v", *ev)
|
|
if err := ev.Do(ctx); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
|
|
var filename string
|
|
started := false
|
|
endTimeout := time.Now().Add(30 * time.Second)
|
|
startTimeout := time.Now().Add(5 * time.Second)
|
|
tick := 500 * time.Millisecond
|
|
for {
|
|
time.Sleep(tick)
|
|
if time.Now().After(endTimeout) {
|
|
return "", fmt.Errorf("timeout while downloading in %q", dir)
|
|
}
|
|
|
|
if !started && time.Now().After(startTimeout) {
|
|
return "", fmt.Errorf("downloading in %q took too long to start", dir)
|
|
}
|
|
entries, err := ioutil.ReadDir(dir)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var fileEntries []string
|
|
for _, v := range entries {
|
|
if v.IsDir() {
|
|
continue
|
|
}
|
|
if v.Name() == ".lastdone" {
|
|
continue
|
|
}
|
|
fileEntries = append(fileEntries, v.Name())
|
|
}
|
|
if len(fileEntries) < 1 {
|
|
continue
|
|
}
|
|
if !started {
|
|
if len(fileEntries) > 0 {
|
|
started = true
|
|
}
|
|
continue
|
|
}
|
|
if len(fileEntries) > 1 {
|
|
return "", fmt.Errorf("more than one file (%d) in download dir %q", len(fileEntries), dir)
|
|
}
|
|
if !strings.HasSuffix(fileEntries[0], ".crdownload") {
|
|
// download is over
|
|
filename = fileEntries[0]
|
|
break
|
|
}
|
|
}
|
|
|
|
if !doMarkDone {
|
|
return filename, nil
|
|
}
|
|
|
|
if err := markDone(dir, location); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return filename, nil
|
|
}
|
|
|
|
mvDl := func(dlFile string) func(ctx context.Context) error {
|
|
return func(ctx context.Context) error {
|
|
dir, err := ioutil.TempDir(s.dlDir, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.Rename(filepath.Join(s.dlDir, dlFile), filepath.Join(dir, dlFile)); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
dlAndMove := func(ctx context.Context, location string, doMarkDone bool) error {
|
|
var err error
|
|
dlFile, err := download(ctx, location, doMarkDone)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return mvDl(dlFile)(ctx)
|
|
}
|
|
|
|
dlAndMoveW := func() func(ctx context.Context) error {
|
|
return func(ctx context.Context) error {
|
|
var location string
|
|
if err := chromedp.Location(&location).Do(ctx); err != nil {
|
|
return err
|
|
}
|
|
return dlAndMove(ctx, location, false)
|
|
}
|
|
}
|
|
|
|
firstNav := func(ctx context.Context) error {
|
|
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
|
|
chromedp.Sleep(500 * time.Millisecond).Do(ctx)
|
|
chromedp.KeyEvent("\n").Do(ctx)
|
|
chromedp.Sleep(500 * time.Millisecond).Do(ctx)
|
|
return nil
|
|
}
|
|
|
|
navRight := func(ctx context.Context) error {
|
|
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
|
|
chromedp.Sleep(5000 * time.Millisecond).Do(ctx)
|
|
chromedp.WaitReady("body", chromedp.ByQuery)
|
|
return nil
|
|
}
|
|
|
|
navLeft := func(ctx context.Context) error {
|
|
chromedp.KeyEvent(kb.ArrowLeft).Do(ctx)
|
|
chromedp.Sleep(5000 * time.Millisecond).Do(ctx)
|
|
chromedp.WaitReady("body", chromedp.ByQuery)
|
|
return nil
|
|
}
|
|
|
|
navN := func(direction string, N int) func(context.Context) error {
|
|
n := 0
|
|
return func(ctx context.Context) error {
|
|
if direction != "left" && direction != "right" {
|
|
return errors.New("wrong direction, pun intended")
|
|
}
|
|
if N == 0 {
|
|
return nil
|
|
}
|
|
var location string
|
|
lookingForLastDone := true
|
|
if s.lastDone == "" {
|
|
lookingForLastDone = false
|
|
}
|
|
for {
|
|
if N > 0 && n >= N {
|
|
break
|
|
}
|
|
if direction == "right" {
|
|
if err := navRight(ctx); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
if err := navLeft(ctx); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := chromedp.Location(&location).Do(ctx); err != nil {
|
|
return err
|
|
}
|
|
if lookingForLastDone {
|
|
if location == s.lastDone {
|
|
lookingForLastDone = false
|
|
}
|
|
println(location, " ALREADY DONE")
|
|
continue
|
|
}
|
|
|
|
// TODO(mpl): deal with getting the very last photo to properly exit that loop when N < 0.
|
|
if err := dlAndMove(ctx, location, true); err != nil {
|
|
return err
|
|
}
|
|
n++
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
if err := chromedp.Run(ctx,
|
|
chromedp.ActionFunc(navEnd),
|
|
page.SetDownloadBehavior(page.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir),
|
|
// TODO(mpl): add policy func over photo URL, which decides what we do with the downloaded file. default policy is storing it on disk.
|
|
chromedp.Navigate("https://photos.google.com/"),
|
|
chromedp.Sleep(5000*time.Millisecond),
|
|
chromedp.WaitReady("body", chromedp.ByQuery),
|
|
chromedp.ActionFunc(func(ctx context.Context) error {
|
|
log.Printf("body is ready")
|
|
return nil
|
|
}),
|
|
// For some reason, I need to do a pagedown before, for the end key to work...
|
|
chromedp.KeyEvent(kb.PageDown),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.End),
|
|
chromedp.Sleep(5000*time.Millisecond),
|
|
// TODO(mpl): do it the smart(er) way: nav right in photo view until the URL does not change anymore. Or something.
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.KeyEvent(kb.ArrowRight),
|
|
chromedp.Sleep(500*time.Millisecond),
|
|
chromedp.ActionFunc(firstNav),
|
|
chromedp.ActionFunc(dlAndMoveW()),
|
|
chromedp.ActionFunc(navN("left", *nItemsFlag-1)),
|
|
); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
fmt.Println("OK")
|
|
|
|
// Next: keys
|
|
// https://github.com/chromedp/chromedp/issues/400
|
|
// https://godoc.org/github.com/chromedp/chromedp/kb
|
|
|
|
_ = firstNav
|
|
|
|
}
|
|
|
|
type Session struct {
|
|
parentContext context.Context
|
|
parentCancel context.CancelFunc
|
|
dlDir string
|
|
profileDir string
|
|
lastDone string
|
|
}
|
|
|
|
func getLastDone(dlDir string) (string, error) {
|
|
data, err := ioutil.ReadFile(filepath.Join(dlDir, ".lastdone"))
|
|
if err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return "", err
|
|
}
|
|
return "", nil
|
|
}
|
|
return string(data), nil
|
|
}
|
|
|
|
func NewSession() (*Session, error) {
|
|
var dir string
|
|
if *devFlag {
|
|
dir = filepath.Join(os.TempDir(), "gphotos-cdp")
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
var err error
|
|
dir, err = ioutil.TempDir("", "gphotos-cdp")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
dlDir := *dlDirFlag
|
|
if dlDir == "" {
|
|
dlDir = filepath.Join(os.Getenv("HOME"), "Downloads", "gphotos-cdp")
|
|
}
|
|
if err := os.MkdirAll(dlDir, 0700); err != nil {
|
|
return nil, err
|
|
}
|
|
lastDone, err := getLastDone(dlDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
s := &Session{
|
|
profileDir: dir,
|
|
dlDir: dlDir,
|
|
lastDone: lastDone,
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func (s *Session) NewContext() (context.Context, context.CancelFunc) {
|
|
ctx, cancel := chromedp.NewExecAllocator(context.Background(),
|
|
chromedp.NoFirstRun,
|
|
chromedp.NoDefaultBrowserCheck,
|
|
chromedp.UserDataDir(s.profileDir),
|
|
|
|
chromedp.Flag("disable-background-networking", true),
|
|
chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"),
|
|
chromedp.Flag("disable-background-timer-throttling", true),
|
|
chromedp.Flag("disable-backgrounding-occluded-windows", true),
|
|
chromedp.Flag("disable-breakpad", true),
|
|
chromedp.Flag("disable-client-side-phishing-detection", true),
|
|
chromedp.Flag("disable-default-apps", true),
|
|
chromedp.Flag("disable-dev-shm-usage", true),
|
|
chromedp.Flag("disable-extensions", true),
|
|
chromedp.Flag("disable-features", "site-per-process,TranslateUI,BlinkGenPropertyTrees"),
|
|
chromedp.Flag("disable-hang-monitor", true),
|
|
chromedp.Flag("disable-ipc-flooding-protection", true),
|
|
chromedp.Flag("disable-popup-blocking", true),
|
|
chromedp.Flag("disable-prompt-on-repost", true),
|
|
chromedp.Flag("disable-renderer-backgrounding", true),
|
|
chromedp.Flag("disable-sync", true),
|
|
chromedp.Flag("force-color-profile", "srgb"),
|
|
chromedp.Flag("metrics-recording-only", true),
|
|
chromedp.Flag("safebrowsing-disable-auto-update", true),
|
|
chromedp.Flag("enable-automation", true),
|
|
chromedp.Flag("password-store", "basic"),
|
|
chromedp.Flag("use-mock-keychain", true),
|
|
)
|
|
s.parentContext = ctx
|
|
s.parentCancel = cancel
|
|
ctx, cancel = chromedp.NewContext(s.parentContext)
|
|
return ctx, cancel
|
|
}
|
|
|
|
func (s *Session) Shutdown() {
|
|
s.parentCancel()
|
|
}
|
|
|
|
func (s *Session) cleanDlDir() error {
|
|
if s.dlDir == "" {
|
|
return nil
|
|
}
|
|
entries, err := ioutil.ReadDir(s.dlDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, v := range entries {
|
|
if v.IsDir() {
|
|
continue
|
|
}
|
|
if err := os.Remove(filepath.Join(s.dlDir, v.Name())); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|