gphotos-cdp/main.go

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/chromedp/cdproto/input"
	"github.com/chromedp/cdproto/page"
	"github.com/chromedp/chromedp"
	"github.com/chromedp/chromedp/kb"
)

var (
	nItemsFlag = flag.Int("n", -1, "number of items to download. If negative, get them all.")
	devFlag    = flag.Bool("dev", false, "dev mode. we reuse the same session dir (/tmp/gphotos-cdp), so we don't have to auth at every run.")
	dlDirFlag  = flag.String("dldir", "", "where to (temporarily) write the downloads. defaults to $HOME/Downloads/gphotos-cdp.")
)

func main() {
	flag.Parse()
	if *nItemsFlag == 0 {
		return
	}
	s, err := NewSession()
	if err != nil {
		log.Fatal(err)
	}
	defer s.Shutdown()

	log.Printf("Session Dir: %v", s.profileDir)

	if err := s.cleanDlDir(); err != nil {
		log.Fatal(err)
	}

	ctx, cancel := s.NewContext()
	defer cancel()

	var outerBefore string

	// login phase
	if err := chromedp.Run(ctx,
		chromedp.ActionFunc(func(ctx context.Context) error {
			log.Printf("pre-navigate")
			return nil
		}),
		chromedp.Navigate("https://photos.google.com/"),
		// chromedp.Sleep(30000*time.Millisecond),
		chromedp.Sleep(5000*time.Millisecond),
		chromedp.ActionFunc(func(ctx context.Context) error {
			log.Printf("post-navigate")
			return nil
		}),
		chromedp.OuterHTML("html>body", &outerBefore),
		chromedp.ActionFunc(func(ctx context.Context) error {
			log.Printf("Source is %d bytes", len(outerBefore))
			return nil
		}),
	); err != nil {
		log.Fatal(err)
	}

	navEnd := func(ctx context.Context) error {
		keyEnd, ok := kb.Keys['\u0305']
		if !ok {
			return errors.New("no End key")
		}

		down := input.DispatchKeyEventParams{
			Key:  keyEnd.Key,
			Code: keyEnd.Code,
			// Some github issue says to remove NativeVirtualKeyCode, but it does not change anything.
			NativeVirtualKeyCode:  keyEnd.Native,
			WindowsVirtualKeyCode: keyEnd.Windows,
			Type:                  input.KeyDown,
		}
		if runtime.GOOS == "darwin" {
			down.NativeVirtualKeyCode = 0
		}
		up := down
		up.Type = input.KeyUp

		for _, ev := range []*input.DispatchKeyEventParams{&down, &up} {
			log.Printf("Event: %+v", *ev)
			if err := ev.Do(ctx); err != nil {
				return err
			}
		}
		time.Sleep(5 * time.Second)
		return nil
	}

	markDone := func(dldir, location string) error {
		println("LOCATION: ", location)
		// TODO(mpl): back up .lastdone before overwriting it, in case writing it fails.
		if err := ioutil.WriteFile(filepath.Join(dldir, ".lastdone"), []byte(location), 0600); err != nil {
			return err
		}
		return nil
	}

	download := func(ctx context.Context, location string) (string, error) {
		dir := s.dlDir
		keyD, ok := kb.Keys['D']
		if !ok {
			log.Fatal("NO D KEY")
		}

		down := input.DispatchKeyEventParams{
			Key:  keyD.Key,
			Code: keyD.Code,
			// Some github issue says to remove NativeVirtualKeyCode, but it does not change anything.
			NativeVirtualKeyCode:  keyD.Native,
			WindowsVirtualKeyCode: keyD.Windows,
			Type:                  input.KeyDown,
			Modifiers:             input.ModifierShift,
		}
		if runtime.GOOS == "darwin" {
			down.NativeVirtualKeyCode = 0
		}
		up := down
		up.Type = input.KeyUp

		for _, ev := range []*input.DispatchKeyEventParams{&down, &up} {
			log.Printf("Event: %+v", *ev)
			if err := ev.Do(ctx); err != nil {
				return "", err
			}
		}

		var filename string
		started := false
		endTimeout := time.Now().Add(30 * time.Second)
		startTimeout := time.Now().Add(5 * time.Second)
		tick := 500 * time.Millisecond
		for {
			time.Sleep(tick)
			if time.Now().After(endTimeout) {
				return "", fmt.Errorf("timeout while downloading in %q", dir)
			}

			if !started && time.Now().After(startTimeout) {
				return "", fmt.Errorf("downloading in %q took too long to start", dir)
			}
			entries, err := ioutil.ReadDir(dir)
			if err != nil {
				return "", err
			}
			var fileEntries []string
			for _, v := range entries {
				if v.IsDir() {
					continue
				}
				fileEntries = append(fileEntries, v.Name())
			}
			if len(fileEntries) < 1 {
				continue
			}
			if !started {
				if len(fileEntries) > 0 {
					started = true
				}
				continue
			}
			if len(fileEntries) > 1 {
				return "", fmt.Errorf("more than one file (%d) in download dir %q", len(fileEntries), dir)
			}
			if !strings.HasSuffix(fileEntries[0], ".crdownload") {
				// download is over
				filename = fileEntries[0]
				break
			}
		}

		if err := markDone(dir, location); err != nil {
			return "", err
		}

		return filename, nil
	}

	mvDl := func(dlFile string) func(ctx context.Context) error {
		return func(ctx context.Context) error {
			dir, err := ioutil.TempDir(s.dlDir, "")
			if err != nil {
				return err
			}
			if err := os.Rename(filepath.Join(s.dlDir, dlFile), filepath.Join(dir, dlFile)); err != nil {
				return err
			}
			return nil
		}
	}

	dlAndMove := func(ctx context.Context, location string) error {
		var err error
		dlFile, err := download(ctx, location)
		if err != nil {
			return err
		}
		return mvDl(dlFile)(ctx)
	}

	firstNav := func(ctx context.Context) error {
		chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
		log.Printf("sent key")
		chromedp.Sleep(500 * time.Millisecond).Do(ctx)
		chromedp.KeyEvent("\n").Do(ctx)
		chromedp.Sleep(500 * time.Millisecond).Do(ctx)
		return nil
	}

	navRight := func(ctx context.Context) error {
		chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
		log.Printf("sent key")
		chromedp.Sleep(5000 * time.Millisecond).Do(ctx)
		chromedp.WaitReady("body", chromedp.ByQuery)
		return nil
	}

	navLeft := func(ctx context.Context) error {
		chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
		log.Printf("sent key")
		chromedp.Sleep(5000 * time.Millisecond).Do(ctx)
		chromedp.WaitReady("body", chromedp.ByQuery)
		return nil
	}

	navN := func(direction string, N int) func(context.Context) error {
		n := 0
		return func(ctx context.Context) error {
			if direction != "left" && direction != "right" {
				return errors.New("wrong direction, pun intended")
			}
			if N == 0 {
				return nil
			}
			var location string
			lookingForLastDone := true
			if s.lastDone == "" {
				lookingForLastDone = false
			}
			for {
				if N > 0 && n >= N {
					break
				}
				if direction == "right" {
					if err := navRight(ctx); err != nil {
						return err
					}
				} else {
					if err := navLeft(ctx); err != nil {
						return err
					}
				}
				if err := chromedp.Location(&location).Do(ctx); err != nil {
					return err
				}
				if lookingForLastDone {
					if location == s.lastDone {
						lookingForLastDone = false
					}
					println(location, " ALREADY DONE")
					continue
				}

				// TODO(mpl): deal with getting the very last photo to properly exit that loop when N < 0.
				if err := dlAndMove(ctx, location); err != nil {
					return err
				}
				n++
			}
			return nil
		}
	}

	if err := chromedp.Run(ctx,
		chromedp.ActionFunc(navEnd),
		page.SetDownloadBehavior(page.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir),
		// TODO(mpl): add policy func over photo URL, which decides what we do with the downloaded file. default policy is storing it on disk.
		chromedp.Navigate("https://photos.google.com/"),
		chromedp.Sleep(5000*time.Millisecond),
		chromedp.WaitReady("body", chromedp.ByQuery),
		chromedp.ActionFunc(func(ctx context.Context) error {
			log.Printf("body is ready")
			return nil
		}),
		// For some reason, I need to do a pagedown before, for the end key to work...
		chromedp.KeyEvent(kb.PageDown),
		chromedp.Sleep(500*time.Millisecond),
		chromedp.KeyEvent(kb.End),
		chromedp.Sleep(5000*time.Millisecond),
		// TODO(mpl): do it the smart(er) way: nav right in photo view until the URL does not change anymore. Or something.
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.KeyEvent(kb.ArrowRight),
		chromedp.ActionFunc(firstNav),
		chromedp.ActionFunc(dlAndMove),
		chromedp.ActionFunc(navN("left", *nItemsFlag-1)),
	); err != nil {
		log.Fatal(err)
	}
	fmt.Println("OK")

	// Next: keys
	// https://github.com/chromedp/chromedp/issues/400
	// https://godoc.org/github.com/chromedp/chromedp/kb

	_ = firstNav

}

type Session struct {
	parentContext context.Context
	parentCancel  context.CancelFunc
	dlDir         string
	profileDir    string
	lastDone      string
}

func getLastDone(dlDir string) (string, error) {
	data, err := ioutil.ReadFile(filepath.Join(dlDir, ".lastdone"))
	if err != nil {
		if !os.IsNotExist(err) {
			return "", err
		}
		return "", nil
	}
	return string(data), nil
}

func NewSession() (*Session, error) {
	var dir string
	if *devFlag {
		dir = filepath.Join(os.TempDir(), "gphotos-cdp")
		if err := os.MkdirAll(dir, 0700); err != nil {
			return nil, err
		}
	} else {
		var err error
		dir, err = ioutil.TempDir("", "gphotos-cdp")
		if err != nil {
			return nil, err
		}
	}
	dlDir := *dlDirFlag
	if dlDir == "" {
		dlDir = filepath.Join(os.Getenv("HOME"), "Downloads", "gphotos-cdp")
	}
	if err := os.MkdirAll(dlDir, 0700); err != nil {
		return nil, err
	}
	lastDone, err := getLastDone(dlDir)
	if err != nil {
		return nil, err
	}
	s := &Session{
		profileDir: dir,
		dlDir:      dlDir,
		lastDone:   lastDone,
	}
	return s, nil
}

func (s *Session) NewContext() (context.Context, context.CancelFunc) {
	ctx, cancel := chromedp.NewExecAllocator(context.Background(),
		chromedp.NoFirstRun,
		chromedp.NoDefaultBrowserCheck,
		chromedp.UserDataDir(s.profileDir),

		chromedp.Flag("disable-background-networking", true),
		chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"),
		chromedp.Flag("disable-background-timer-throttling", true),
		chromedp.Flag("disable-backgrounding-occluded-windows", true),
		chromedp.Flag("disable-breakpad", true),
		chromedp.Flag("disable-client-side-phishing-detection", true),
		chromedp.Flag("disable-default-apps", true),
		chromedp.Flag("disable-dev-shm-usage", true),
		chromedp.Flag("disable-extensions", true),
		chromedp.Flag("disable-features", "site-per-process,TranslateUI,BlinkGenPropertyTrees"),
		chromedp.Flag("disable-hang-monitor", true),
		chromedp.Flag("disable-ipc-flooding-protection", true),
		chromedp.Flag("disable-popup-blocking", true),
		chromedp.Flag("disable-prompt-on-repost", true),
		chromedp.Flag("disable-renderer-backgrounding", true),
		chromedp.Flag("disable-sync", true),
		chromedp.Flag("force-color-profile", "srgb"),
		chromedp.Flag("metrics-recording-only", true),
		chromedp.Flag("safebrowsing-disable-auto-update", true),
		chromedp.Flag("enable-automation", true),
		chromedp.Flag("password-store", "basic"),
		chromedp.Flag("use-mock-keychain", true),
	)
	s.parentContext = ctx
	s.parentCancel = cancel
	ctx, cancel = chromedp.NewContext(s.parentContext)
	return ctx, cancel
}

func (s *Session) Shutdown() {
	s.parentCancel()
}

func (s *Session) cleanDlDir() error {
	if s.dlDir == "" {
		return nil
	}
	entries, err := ioutil.ReadDir(s.dlDir)
	if err != nil {
		return err
	}
	for _, v := range entries {
		if v.IsDir() {
			continue
		}
		if err := os.Remove(filepath.Join(s.dlDir, v.Name())); err != nil {
			return err
		}
	}
	return nil
}