Archives
/
gphotos-cdp
mirror of https://github.com/perkeep/gphotos-cdp
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactored; funcs and session methods

Browse Source
pull/2/merge
mpl 5 years ago
parent a2e303781b
commit 886b757cbc
1 changed files with 271 additions and 269 deletions
Show Stats Download Patch File Download Diff File

384
main.go
Unescape Escape View File

@ -26,6 +26,7 @@ var (
dlDirFlag = flag.String("dldir", "", "where to (temporarily) write the downloads. defaults to $HOME/Downloads/gphotos-cdp.")
startFlag = flag.String("start", "", "skip all photos until this location is reached. for debugging.")
runFlag = flag.String("run", "", "the program to run on each downloaded item, right after it is dowloaded. It is also the responsibility of that program to remove the downloaded item, if desired.")
verboseFlag = flag.Bool("v", false, "be verbose")
)
// TODO(mpl): in general everywhere, do not rely so much on sleeps. We need
@ -54,12 +55,146 @@ func main() {
ctx, cancel := s.NewContext()
defer cancel()
var outerBefore string
if err := login(ctx); err != nil {
log.Fatal(err)
}
// login phase
if err := chromedp.Run(ctx,
page.SetDownloadBehavior(page.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir),
chromedp.Navigate("https://photos.google.com/"),
chromedp.Sleep(5000*time.Millisecond),
chromedp.WaitReady("body", chromedp.ByQuery),
chromedp.ActionFunc(func(ctx context.Context) error {
if *verboseFlag {
log.Printf("body is ready")
}
return nil
}),
chromedp.ActionFunc(s.firstNav),
chromedp.ActionFunc(s.navN(*nItemsFlag)),
); err != nil {
log.Fatal(err)
}
fmt.Println("OK")
}
type Session struct {
parentContext context.Context
parentCancel context.CancelFunc
dlDir string
profileDir string
lastDone string
}
func getLastDone(dlDir string) (string, error) {
data, err := ioutil.ReadFile(filepath.Join(dlDir, ".lastdone"))
if err != nil {
if !os.IsNotExist(err) {
return "", err
}
return "", nil
}
return string(data), nil
}
func NewSession() (*Session, error) {
var dir string
if *devFlag {
dir = filepath.Join(os.TempDir(), "gphotos-cdp")
if err := os.MkdirAll(dir, 0700); err != nil {
return nil, err
}
} else {
var err error
dir, err = ioutil.TempDir("", "gphotos-cdp")
if err != nil {
return nil, err
}
}
dlDir := *dlDirFlag
if dlDir == "" {
dlDir = filepath.Join(os.Getenv("HOME"), "Downloads", "gphotos-cdp")
}
if err := os.MkdirAll(dlDir, 0700); err != nil {
return nil, err
}
lastDone, err := getLastDone(dlDir)
if err != nil {
return nil, err
}
s := &Session{
profileDir: dir,
dlDir: dlDir,
lastDone: lastDone,
}
return s, nil
}
func (s *Session) NewContext() (context.Context, context.CancelFunc) {
ctx, cancel := chromedp.NewExecAllocator(context.Background(),
chromedp.NoFirstRun,
chromedp.NoDefaultBrowserCheck,
chromedp.UserDataDir(s.profileDir),
chromedp.Flag("disable-background-networking", true),
chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"),
chromedp.Flag("disable-background-timer-throttling", true),
chromedp.Flag("disable-backgrounding-occluded-windows", true),
chromedp.Flag("disable-breakpad", true),
chromedp.Flag("disable-client-side-phishing-detection", true),
chromedp.Flag("disable-default-apps", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-extensions", true),
chromedp.Flag("disable-features", "site-per-process,TranslateUI,BlinkGenPropertyTrees"),
chromedp.Flag("disable-hang-monitor", true),
chromedp.Flag("disable-ipc-flooding-protection", true),
chromedp.Flag("disable-popup-blocking", true),
chromedp.Flag("disable-prompt-on-repost", true),
chromedp.Flag("disable-renderer-backgrounding", true),
chromedp.Flag("disable-sync", true),
chromedp.Flag("force-color-profile", "srgb"),
chromedp.Flag("metrics-recording-only", true),
chromedp.Flag("safebrowsing-disable-auto-update", true),
chromedp.Flag("enable-automation", true),
chromedp.Flag("password-store", "basic"),
chromedp.Flag("use-mock-keychain", true),
)
s.parentContext = ctx
s.parentCancel = cancel
ctx, cancel = chromedp.NewContext(s.parentContext)
return ctx, cancel
}
func (s *Session) Shutdown() {
s.parentCancel()
}
func (s *Session) cleanDlDir() error {
if s.dlDir == "" {
return nil
}
entries, err := ioutil.ReadDir(s.dlDir)
if err != nil {
return err
}
for _, v := range entries {
if v.IsDir() {
continue
}
if err := os.Remove(filepath.Join(s.dlDir, v.Name())); err != nil {
return err
}
}
return nil
}
func login(ctx context.Context) error {
var outerBefore string
return chromedp.Run(ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
if *verboseFlag {
log.Printf("pre-navigate")
}
return nil
}),
chromedp.Navigate("https://photos.google.com/"),
@ -73,19 +208,22 @@ func main() {
return nil
}),
chromedp.ActionFunc(func(ctx context.Context) error {
if *verboseFlag {
log.Printf("post-navigate")
}
return nil
}),
chromedp.OuterHTML("html>body", &outerBefore),
chromedp.ActionFunc(func(ctx context.Context) error {
if *verboseFlag {
log.Printf("Source is %d bytes", len(outerBefore))
}
return nil
}),
); err != nil {
log.Fatal(err)
}
)
}
firstNav := func(ctx context.Context) error {
func (s Session) firstNav(ctx context.Context) error {
if *startFlag != "" {
chromedp.Navigate(*startFlag).Do(ctx)
chromedp.WaitReady("body", chromedp.ByQuery).Do(ctx)
@ -123,28 +261,42 @@ func main() {
prevLocation = location
}
return nil
}
func doRun(filePath string) error {
if *runFlag == "" {
return nil
}
return exec.Command(*runFlag, filePath).Run()
}
func navLeft(ctx context.Context) error {
chromedp.KeyEvent(kb.ArrowLeft).Do(ctx)
chromedp.WaitReady("body", chromedp.ByQuery)
chromedp.Sleep(1 * time.Second).Do(ctx)
return nil
}
markDone := func(dldir, location string) error {
println("LOCATION: ", location)
func markDone(dldir, location string) error {
if *verboseFlag {
log.Printf("Marking %v as done", location)
}
// TODO(mpl): back up .lastdone before overwriting it, in case writing it fails.
if err := ioutil.WriteFile(filepath.Join(dldir, ".lastdone"), []byte(location), 0600); err != nil {
return err
}
return nil
}
}
download := func(ctx context.Context, location string) (string, error) {
dir := s.dlDir
func startDownload(ctx context.Context) error {
keyD, ok := kb.Keys['D']
if !ok {
log.Fatal("NO D KEY")
return errors.New("no D key")
}
down := input.DispatchKeyEventParams{
Key: keyD.Key,
Code: keyD.Code,
// Some github issue says to remove NativeVirtualKeyCode, but it does not change anything.
NativeVirtualKeyCode: keyD.Native,
WindowsVirtualKeyCode: keyD.Windows,
Type: input.KeyDown,
@ -157,11 +309,21 @@ func main() {
up.Type = input.KeyUp
for _, ev := range []*input.DispatchKeyEventParams{&down, &up} {
// log.Printf("Event: %+v", *ev)
if *verboseFlag {
log.Printf("Event: %+v", *ev)
}
if err := ev.Do(ctx); err != nil {
return "", err
return err
}
}
return nil
}
func (s Session) download(ctx context.Context, location string) (string, error) {
if err := startDownload(ctx); err != nil {
return "", err
}
var filename string
started := false
@ -173,13 +335,13 @@ func main() {
// TODO(mpl): download starts late if it's a video. figure out if dl can only
// start after video has started playing or something like that?
if !started && time.Now().After(timeout) {
return "", fmt.Errorf("downloading in %q took too long to start", dir)
return "", fmt.Errorf("downloading in %q took too long to start", s.dlDir)
}
if started && time.Now().After(timeout) {
return "", fmt.Errorf("timeout while downloading in %q", dir)
return "", fmt.Errorf("timeout while downloading in %q", s.dlDir)
}
entries, err := ioutil.ReadDir(dir)
entries, err := ioutil.ReadDir(s.dlDir)
if err != nil {
return "", err
}
@ -197,7 +359,7 @@ func main() {
continue
}
if len(fileEntries) > 1 {
return "", fmt.Errorf("more than one file (%d) in download dir %q", len(fileEntries), dir)
return "", fmt.Errorf("more than one file (%d) in download dir %q", len(fileEntries), s.dlDir)
}
if !started {
if len(fileEntries) > 0 {
@ -218,15 +380,14 @@ func main() {
}
}
if err := markDone(dir, location); err != nil {
if err := markDone(s.dlDir, location); err != nil {
return "", err
}
return filename, nil
}
}
mvDl := func(dlFile, location string) func(ctx context.Context) (string, error) {
return func(ctx context.Context) (string, error) {
func (s Session) moveDownload(ctx context.Context, dlFile, location string) (string, error) {
parts := strings.Split(location, "/")
if len(parts) < 5 {
return "", fmt.Errorf("not enough slash separated parts in location %v: %d", location, len(parts))
@ -240,45 +401,19 @@ func main() {
return "", err
}
return newFile, nil
}
}
}
dlAndMove := func(ctx context.Context, location string) (string, error) {
var err error
dlFile, err := download(ctx, location)
func (s Session) dlAndMove(ctx context.Context, location string) (string, error) {
dlFile, err := s.download(ctx, location)
if err != nil {
return "", err
}
return mvDl(dlFile, location)(ctx)
}
doRun := func(ctx context.Context, filePath string) error {
if *runFlag == "" {
return nil
}
return exec.Command(*runFlag, filePath).Run()
}
navRight := func(ctx context.Context) error {
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
chromedp.WaitReady("body", chromedp.ByQuery)
chromedp.Sleep(1 * time.Second).Do(ctx)
return nil
}
navLeft := func(ctx context.Context) error {
chromedp.KeyEvent(kb.ArrowLeft).Do(ctx)
chromedp.WaitReady("body", chromedp.ByQuery)
chromedp.Sleep(1 * time.Second).Do(ctx)
return nil
}
return s.moveDownload(ctx, dlFile, location)
}
navN := func(direction string, N int) func(context.Context) error {
n := 0
func (s Session) navN(N int) func(context.Context) error {
return func(ctx context.Context) error {
if direction != "left" && direction != "right" {
return errors.New("wrong direction, pun intended")
}
n := 0
if N == 0 {
return nil
}
@ -292,155 +427,22 @@ func main() {
break
}
prevLocation = location
filePath, err := dlAndMove(ctx, location)
filePath, err := s.dlAndMove(ctx, location)
if err != nil {
return err
}
// TODO(mpl): do run in a go routine?
if err := doRun(ctx, filePath); err != nil {
if err := doRun(filePath); err != nil {
return err
}
n++
if N > 0 && n >= N {
break
}
if direction == "right" {
if err := navRight(ctx); err != nil {
return err
}
} else {
if err := navLeft(ctx); err != nil {
return err
}
}
}
return nil
}
}
if err := chromedp.Run(ctx,
page.SetDownloadBehavior(page.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir),
chromedp.Navigate("https://photos.google.com/"),
chromedp.Sleep(5000*time.Millisecond),
chromedp.WaitReady("body", chromedp.ByQuery),
chromedp.ActionFunc(func(ctx context.Context) error {
log.Printf("body is ready")
return nil
}),
chromedp.ActionFunc(firstNav),
chromedp.ActionFunc(navN("left", *nItemsFlag)),
); err != nil {
log.Fatal(err)
}
fmt.Println("OK")
}
type Session struct {
parentContext context.Context
parentCancel context.CancelFunc
dlDir string
profileDir string
lastDone string
}
func getLastDone(dlDir string) (string, error) {
data, err := ioutil.ReadFile(filepath.Join(dlDir, ".lastdone"))
if err != nil {
if !os.IsNotExist(err) {
return "", err
}
return "", nil
}
return string(data), nil
}
func NewSession() (*Session, error) {
var dir string
if *devFlag {
dir = filepath.Join(os.TempDir(), "gphotos-cdp")
if err := os.MkdirAll(dir, 0700); err != nil {
return nil, err
}
} else {
var err error
dir, err = ioutil.TempDir("", "gphotos-cdp")
if err != nil {
return nil, err
}
}
dlDir := *dlDirFlag
if dlDir == "" {
dlDir = filepath.Join(os.Getenv("HOME"), "Downloads", "gphotos-cdp")
}
if err := os.MkdirAll(dlDir, 0700); err != nil {
return nil, err
}
lastDone, err := getLastDone(dlDir)
if err != nil {
return nil, err
}
s := &Session{
profileDir: dir,
dlDir: dlDir,
lastDone: lastDone,
}
return s, nil
}
func (s *Session) NewContext() (context.Context, context.CancelFunc) {
ctx, cancel := chromedp.NewExecAllocator(context.Background(),
chromedp.NoFirstRun,
chromedp.NoDefaultBrowserCheck,
chromedp.UserDataDir(s.profileDir),
chromedp.Flag("disable-background-networking", true),
chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"),
chromedp.Flag("disable-background-timer-throttling", true),
chromedp.Flag("disable-backgrounding-occluded-windows", true),
chromedp.Flag("disable-breakpad", true),
chromedp.Flag("disable-client-side-phishing-detection", true),
chromedp.Flag("disable-default-apps", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-extensions", true),
chromedp.Flag("disable-features", "site-per-process,TranslateUI,BlinkGenPropertyTrees"),
chromedp.Flag("disable-hang-monitor", true),
chromedp.Flag("disable-ipc-flooding-protection", true),
chromedp.Flag("disable-popup-blocking", true),
chromedp.Flag("disable-prompt-on-repost", true),
chromedp.Flag("disable-renderer-backgrounding", true),
chromedp.Flag("disable-sync", true),
chromedp.Flag("force-color-profile", "srgb"),
chromedp.Flag("metrics-recording-only", true),
chromedp.Flag("safebrowsing-disable-auto-update", true),
chromedp.Flag("enable-automation", true),
chromedp.Flag("password-store", "basic"),
chromedp.Flag("use-mock-keychain", true),
)
s.parentContext = ctx
s.parentCancel = cancel
ctx, cancel = chromedp.NewContext(s.parentContext)
return ctx, cancel
}
func (s *Session) Shutdown() {
s.parentCancel()
}
func (s *Session) cleanDlDir() error {
if s.dlDir == "" {
return nil
}
entries, err := ioutil.ReadDir(s.dlDir)
if err != nil {
return err
}
for _, v := range entries {
if v.IsDir() {
continue
}
if err := os.Remove(filepath.Join(s.dlDir, v.Name())); err != nil {
return err
}
}
return nil
}

Write Preview
Loading…
Cancel
Save