From c202f2eca32e1ab2e313417168351df1c58ee062 Mon Sep 17 00:00:00 2001 From: Mitch Riedstra Date: Wed, 4 Aug 2021 23:53:36 -0400 Subject: More major changes. Web UI works. Downloading games works. Status works. extractFile needs work --- steam/delete.go | 41 ++++++++++++ steam/extract.go | 154 +++++++++++++++++++++++++++++++++++++++++- steam/extract_http.go | 34 ++++++++++ steam/game.go | 9 +++ steam/http.go | 61 ----------------- steam/package.go | 181 ++++++++++++++++---------------------------------- steam/status.go | 82 +++++++++++++++++++++++ steam/steam.go | 38 +++++++---- 8 files changed, 401 insertions(+), 199 deletions(-) create mode 100644 steam/delete.go create mode 100644 steam/extract_http.go delete mode 100644 steam/http.go (limited to 'steam') diff --git a/steam/delete.go b/steam/delete.go new file mode 100644 index 0000000..486b047 --- /dev/null +++ b/steam/delete.go @@ -0,0 +1,41 @@ +package steam + +import ( + "os" + "path/filepath" +) + +// Delete removes all of the game files and the ACF +func (l *Library) Delete(game string) error { + g, ok := l.games[game] + if !ok { + return E_GameDoesNotExist + } + + j := newJob("delete", g) + defer j.done() + + l.status.addJob(j) + + acf, err := FindACF(l.folder, game) + if err != nil { + j.addError(err) + return err + } + if err := os.Remove(acf); err != nil { + j.addError(err) + return err + } + + err = os.RemoveAll(filepath.Join(l.folder, "common", g.Name)) + if err != nil { + j.addError(err) + return err + } + + l.m.Lock() + delete(l.games, game) + l.m.Unlock() + + return nil +} diff --git a/steam/extract.go b/steam/extract.go index f73d5f7..9a7a930 100644 --- a/steam/extract.go +++ b/steam/extract.go @@ -1,15 +1,90 @@ package steam import ( + "archive/tar" "errors" "fmt" "io" + "net/url" + "os" + "path/filepath" + "strings" "time" ) // how often are we going to be updating our status information? const updateEveryNBytes = 10 * 1024 * 1024 // 10mb +// ExtractSmart attempts to discover what kind of resource is behind uri +// and extract it appropriately. It may fail with E_BadURI. +// +// For example the following forms are accepted: +// +// ExtractSmart("http://127.0.0.1/some-archive") +// ExtractSmart("https://example.com/some-archive") +// ExtractSmart("file:///some/local/file/path/to/archive.tar") +// ExtractSmart("/direct/path/to/archive.tar") +// ExtractSmart("C:\Users\user\Downloads\archive.tar") +func (l *Library) ExtractSmart(uri string) (*Game, error) { + if strings.HasPrefix(uri, "http") { + _, err := url.Parse(uri) + if err == nil { + return l.ExtractHTTP(uri) + } + } else if strings.HasPrefix(uri, "file") { + u, err := url.Parse(uri) + if err == nil { + return l.ExtractFile(u.Path) + } + } else if _, err := os.Stat(uri); err == nil { + return l.ExtractFile(uri) + } + + return nil, E_BadURI +} + +// ExtractFile is a wrapper around Extract that handles an HTTP endpoint. +// this spawns an "extractFile" on the library. Status will be updated there +// as this goes along. Non fatal and fatal errors will be populated there +func (l *Library) ExtractFile(fn string) (*Game, error) { + g := &Game{} + j := newJob("extractFile", g) + defer j.done() + + l.status.addJob(j) + + fi, err := os.Stat(fn) + if err != nil { + j.addError(err) + return g, err + } + j.setSize(fi.Size()) + + fh, err := os.Open(fn) + if err != nil { + j.addError(err) + return g, err + } + + return l.extractUpdate(j, g, fh) +} + +// Extract will read a tarball from the io.Reader and install the game into +// the current library path. This offers no visibility into the progress, +// as it does not update the job status on the progress, though it will +// populate errors. +// +// Most callers will want to use ExtractHTTP or ExtractFile instead +func (l *Library) Extract(r io.Reader) (*Game, error) { + g := &Game{LibraryPath: l.folder} + j := newJob("extract", g) + defer j.done() + + l.status.addJob(j) + + return l.extractPrimitive(j, g, r) +} + // extractUpdate takes care of updating the job as it goes along at updateEveryNBytes // it will be reported back to the Job's status. func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, error) { @@ -43,7 +118,8 @@ func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, erro j.setTransferred(total) // rate in bytes/sec - rate := total / int64(time.Since(*j.StartTime()).Seconds()) + // rate := total / int64(time.Since(*j.StartTime()).Seconds()) + rate := float64(total) / float64(time.Since(*j.StartTime()).Seconds()) estSize := j.GetSize() @@ -52,7 +128,8 @@ func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, erro continue } - remaining := *estSize - total + // remaining := *estSize - total + remaining := float64(*estSize - total) j.setETA(time.Duration((remaining / rate) / 1000 / 1000 / 1000)) } @@ -64,3 +141,76 @@ func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, erro return g, err } + +func (l *Library) extractPrimitive(j *Job, g *Game, r io.Reader) (*Game, error) { + treader := tar.NewReader(r) + + for { + hdr, err := treader.Next() + if err == io.EOF { + // We've reached the end! Whoee + break + } + if err != nil { + j.addError(err) + return nil, err + } + + fileName := filepath.ToSlash(hdr.Name) + + if g.Name == "" { + s := strings.Split(fileName, "/") + if len(s) >= 2 { + g.Name = s[1] + } + } + + fileName = filepath.Join(l.folder, fileName) + + info := hdr.FileInfo() + if info.IsDir() { + // I don't like hard-coded permissions but it + // it helps with overall platform compatibility + err = os.MkdirAll(fileName, defaultDirectoryMode) + if err != nil { + j.addError(err) + return nil, err + } + + continue + } + + err = os.MkdirAll(filepath.Dir(fileName), defaultDirectoryMode) + if err != nil { + j.addError(err) + return nil, err + } + + // Create a file handle to work with + f, err := os.OpenFile(fileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, + defaultFileMode) + if err != nil { + j.addError(err) + return nil, err + } + if _, err := io.Copy(f, treader); err != nil { + j.addError(err) + f.Close() + return nil, err + } + f.Close() + + } + + err := g.SetSizeInfo() + if err != nil { + j.addError(err) + return nil, err + } + + l.m.Lock() + l.games[g.Name] = g + l.m.Unlock() + + return g, nil +} diff --git a/steam/extract_http.go b/steam/extract_http.go new file mode 100644 index 0000000..6884bee --- /dev/null +++ b/steam/extract_http.go @@ -0,0 +1,34 @@ +package steam + +import ( + "fmt" + "net/http" + "strconv" +) + +// ExtractHTTP is a wrapper around Extract that handles an HTTP endpoint. +// this spawns an "extractHTTP" on the library. Status will be updated there +// as this goes along. Non fatal and fatal errors will be populated there +func (l *Library) ExtractHTTP(url string) (*Game, error) { + g := &Game{} + j := newJob("extractHTTP", g) + defer j.done() + + l.status.addJob(j) + + resp, err := http.Get(url) + if err != nil { + j.addError(err) + return g, err + } + + estSize, err := strconv.ParseInt(resp.Header.Get("Estimated-size"), 10, 64) + if err != nil { + j.addError(err) + return g, fmt.Errorf("Failed to convert estimated size header: %w", err) + } + + j.setSize(estSize) + + return l.extractUpdate(j, g, resp.Body) +} diff --git a/steam/game.go b/steam/game.go index 0028257..dd4a297 100644 --- a/steam/game.go +++ b/steam/game.go @@ -5,6 +5,15 @@ import ( "path/filepath" ) +// GetSizeBytes returns the size in bytes, calling SetSizeInfo info Size is +// currently == 0 +func (g *Game) GetSizeBytes() int64 { + if g.Size == 0 { + _ = g.SetSizeInfo() + } + return g.Size +} + // GetSize returns the size of a game in a pretty format. If size is 0 // it will call SetSizeInfo before returning func (g *Game) GetSize() string { diff --git a/steam/http.go b/steam/http.go deleted file mode 100644 index df756e4..0000000 --- a/steam/http.go +++ /dev/null @@ -1,61 +0,0 @@ -package steam - -import ( - "fmt" - "net/http" - "os" - "strconv" -) - -// ExtractFile is a wrapper around Extract that handles an HTTP endpoint. -// this spawns an "extractFile" on the library. Status will be updated there -// as this goes along. Non fatal and fatal errors will be populated there -func (l *Library) ExtractFile(fn string) (*Game, error) { - g := &Game{} - j := newJob("extractFile", g) - defer j.done() - - l.status.addJob(j) - - fi, err := os.Stat(fn) - if err != nil { - j.addError(err) - return g, err - } - j.setSize(fi.Size()) - - fh, err := os.Open(fn) - if err != nil { - j.addError(err) - return g, err - } - - return l.extractUpdate(j, g, fh) -} - -// ExtractHTTP is a wrapper around Extract that handles an HTTP endpoint. -// this spawns an "extractHTTP" on the library. Status will be updated there -// as this goes along. Non fatal and fatal errors will be populated there -func (l *Library) ExtractHTTP(url string) (*Game, error) { - g := &Game{} - j := newJob("extractHTTP", g) - defer j.done() - - l.status.addJob(j) - - resp, err := http.Get(url) - if err != nil { - j.addError(err) - return g, err - } - - estSize, err := strconv.ParseInt(resp.Header.Get("Estimated-size"), 10, 64) - if err != nil { - j.addError(err) - return g, fmt.Errorf("Failed to convert estimated size header: %w", err) - } - - j.setSize(estSize) - - return l.extractUpdate(j, g, resp.Body) -} diff --git a/steam/package.go b/steam/package.go index df6f1e8..5b0c618 100644 --- a/steam/package.go +++ b/steam/package.go @@ -2,10 +2,11 @@ package steam import ( "archive/tar" + "errors" + "fmt" "io" - "os" "path/filepath" - "strings" + "time" ) const ( @@ -13,177 +14,111 @@ const ( defaultFileMode = 0644 ) -// Package writes the package to wr, returning errors if any -func (l *Library) Package(game string, wr io.WriteCloser) error { - g, ok := l.games[game] - if !ok { +func (l *Library) Package(game string, wr io.Writer) error { + if _, ok := l.Games()[game]; !ok { return E_GameDoesNotExist } + g := l.Games()[game] j := newJob("package", g) defer j.done() l.status.addJob(j) - acf, err := FindACF(l.folder, g.Name) - if err != nil { - j.addError(err) - return err - } + j.setSize(g.GetSizeBytes()) - twriter := tar.NewWriter(wr) - - paths := []string{ - filepath.Join(l.folder, "common", g.Name), - acf, - } - for _, pth := range paths { - err := filepath.Walk(pth, tarWalkfn(twriter, l.folder)) + // Invert the writer so we can break up the copy and get progress + // information in here + rdr, pwrtr := io.Pipe() + go func() { + err := l.packagePrimitive(j, g, pwrtr) if err != nil { j.addError(err) - return err } - } - - err = twriter.Flush() - if err != nil { - j.addError(err) - return err - } - - err = twriter.Close() - if err != nil { - j.addError(err) - return err - } - - err = wr.Close() - if err != nil { - j.addError(err) - } - return err -} - -// Extract will read a tarball from the io.Reader and install the game into -// the current library path. This offers no visibility into the progress, -// as it does not update the job status on the progress, though it will -// populate errors. -// -// Most callers will want to use ExtractHTTP or ExtractFile instead -func (l *Library) Extract(r io.Reader) (*Game, error) { - g := &Game{LibraryPath: l.folder} - j := newJob("extract", g) - defer j.done() - - l.status.addJob(j) - - return l.extractPrimitive(j, g, r) -} - -func (l *Library) extractPrimitive(j *Job, g *Game, r io.Reader) (*Game, error) { - treader := tar.NewReader(r) + }() + var total int64 for { - hdr, err := treader.Next() + n, err := io.CopyN(wr, rdr, updateEveryNBytes) + if err == io.EOF { - // We've reached the end! Whoee break } if err != nil { j.addError(err) - return nil, err + return err } - fileName := filepath.ToSlash(hdr.Name) + total += n + j.setTransferred(total) - if g.Name == "" { - s := strings.Split(fileName, "/") - if len(s) >= 2 { - g.Name = s[1] - } - } + elapsedSeconds := float64(time.Since(*j.StartTime()).Seconds()) + + rate := float64(total) / elapsedSeconds - fileName = filepath.Join(l.folder, fileName) + fmt.Println("rate in bytes/second: ", formatBytes(int64(rate))) - info := hdr.FileInfo() - if info.IsDir() { - // I don't like hard-coded permissions but it - // it helps with overall platform compatibility - err = os.MkdirAll(fileName, defaultDirectoryMode) - if err != nil { - j.addError(err) - return nil, err - } + estSize := j.GetSize() + if estSize == nil { + j.addError(errors.New("Expected an estimated size, got nil")) continue } - err = os.MkdirAll(filepath.Dir(fileName), defaultDirectoryMode) - if err != nil { - j.addError(err) - return nil, err - } + remainingBytes := float64(*estSize - total) + fmt.Println("remaining bytes: ", formatBytes(int64(remainingBytes))) - // Create a file handle to work with - f, err := os.OpenFile(fileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, - defaultFileMode) - if err != nil { - j.addError(err) - return nil, err - } - if _, err := io.Copy(f, treader); err != nil { - j.addError(err) - f.Close() - return nil, err - } - f.Close() + seconds := (remainingBytes / rate) - } + duration := time.Duration(seconds * 1000 * 1000 * 1000) + fmt.Println("Raw duration: ", duration) - err := g.SetSizeInfo() - if err != nil { - j.addError(err) - return nil, err + j.setETA(duration) } - l.m.Lock() - l.games[g.Name] = g - l.m.Unlock() - - return g, nil + return nil } -// Delete removes all of the game files and the ACF -func (l *Library) Delete(game string) error { - g, ok := l.games[game] - if !ok { - return E_GameDoesNotExist +// Package writes the package to wr, returning errors if any +func (l *Library) packagePrimitive(j *Job, g *Game, wr io.WriteCloser) error { + + acf, err := FindACF(l.folder, g.Name) + if err != nil { + j.addError(err) + return err } - j := newJob("delete", g) - defer j.done() + twriter := tar.NewWriter(wr) - l.status.addJob(j) + paths := []string{ + filepath.Join(l.folder, "common", g.Name), + acf, + } + for _, pth := range paths { + err := filepath.Walk(pth, tarWalkfn(twriter, l.folder)) + if err != nil { + j.addError(err) + return err + } + } - acf, err := FindACF(l.folder, game) + err = twriter.Flush() if err != nil { j.addError(err) return err } - if err := os.Remove(acf); err != nil { + + err = twriter.Close() + if err != nil { j.addError(err) return err } - err = os.RemoveAll(filepath.Join(l.folder, "common", g.Name)) + err = wr.Close() if err != nil { j.addError(err) return err } - l.m.Lock() - delete(l.games, game) - l.m.Unlock() - - return nil + return err } diff --git a/steam/status.go b/steam/status.go index 12f5e3b..9d70655 100644 --- a/steam/status.go +++ b/steam/status.go @@ -1,10 +1,28 @@ package steam import ( + "encoding/json" "sync" "time" + + "fmt" + "os" ) +var debuglogging = false + +func debugLogJob(s string, args ...interface{}) { + if debuglogging { + fmt.Fprintf(os.Stderr, s, args...) + } +} + +func debugLogJobs(s string, args ...interface{}) { + if debuglogging { + fmt.Fprintf(os.Stderr, s, args...) + } +} + // JobStatus provides specific information about an individual job type Job struct { action string @@ -21,10 +39,36 @@ type Job struct { m sync.Mutex } +func (j Job) MarshalJSON() ([]byte, error) { + return json.Marshal( + struct { + Action string `json:"action"` + Target *Game `json:"Target"` + Running bool `json:"Running"` + Start *time.Time `json:"Start"` + Errors []error `json:"Errors"` + + // If applicablle + Size *int64 `json:"Size"` + Transferred *int64 `json:"Transferred"` + Eta *time.Duration `json:"ETA"` + }{ + Action: j.action, + Target: j.target, + Running: j.running, + Start: j.start, + Errors: j.errors, + Size: j.size, + Transferred: j.transferred, + Eta: j.eta, + }) +} + // Action is a short string describing the action, i.e. "packaging", "deleting" func (j *Job) Action() string { j.m.Lock() defer j.m.Unlock() + debugLogJob("Action on: '%s'\n", *j) return j.action } @@ -32,6 +76,7 @@ func (j *Job) Action() string { func (j *Job) Target() *Game { j.m.Lock() defer j.m.Unlock() + debugLogJob("Target on: '%s'\n", *j) return j.target } @@ -39,6 +84,7 @@ func (j *Job) Target() *Game { func (j *Job) IsRunning() bool { j.m.Lock() defer j.m.Unlock() + debugLogJob("IsRunning on: '%s'\n", *j) return j.running } @@ -46,17 +92,20 @@ func (j *Job) IsRunning() bool { func (j *Job) StartTime() *time.Time { j.m.Lock() defer j.m.Unlock() + debugLogJob("StartTime on: '%s'\n", *j) return j.start } func (j *Job) Errors() []error { j.m.Lock() defer j.m.Unlock() + debugLogJob("errors on: '%s'\n", *j) return j.errors } // newJob sets up a job of action for the target Game func newJob(action string, target *Game) *Job { + debugLogJob("New job: '%s' target: '%s'\n", action, target) t := time.Now() return &Job{ action: action, @@ -69,6 +118,7 @@ func newJob(action string, target *Game) *Job { func (j *Job) setSize(size int64) { j.m.Lock() defer j.m.Unlock() + debugLogJob("setSize on: '%s'\n", *j) j.size = &size } @@ -76,12 +126,14 @@ func (j *Job) setSize(size int64) { func (j *Job) GetSize() *int64 { j.m.Lock() defer j.m.Unlock() + debugLogJob("getSize on: '%s'\n", *j) return j.size } func (j *Job) setTransferred(transferred int64) { j.m.Lock() defer j.m.Unlock() + debugLogJob("setTransferred on: '%s'\n", *j) j.transferred = &transferred } @@ -89,6 +141,7 @@ func (j *Job) setTransferred(transferred int64) { func (j *Job) GetTransferred() *int64 { j.m.Lock() defer j.m.Unlock() + debugLogJob("GetTransferred on: '%s'\n", *j) return j.transferred } @@ -96,6 +149,7 @@ func (j *Job) GetTransferred() *int64 { func (j *Job) setETA(d time.Duration) { j.m.Lock() defer j.m.Unlock() + debugLogJob("setETA on: '%s'\n", *j) j.eta = &d } @@ -104,6 +158,7 @@ func (j *Job) setETA(d time.Duration) { func (j *Job) GetETA() *time.Duration { j.m.Lock() defer j.m.Unlock() + debugLogJob("GetETA on: '%s'\n", *j) return j.eta } @@ -118,6 +173,7 @@ func (j *Job) done() { func (j *Job) addError(err error) { j.m.Lock() defer j.m.Unlock() + debugLogJob("add error on: '%s'\n", *j) j.errors = append(j.errors, err) } @@ -131,9 +187,30 @@ type Jobs struct { m sync.Mutex } +func (jobs Jobs) MarshalJSON() ([]byte, error) { + jobs.scan() + return json.Marshal( + struct { + Running []*Job `json:"Running"` + Previous []*Job `json:"Previous"` + }{ + Running: jobs.running, + Previous: jobs.previous, + }) +} + +func (jobs Jobs) String() string { + b, err := json.Marshal(jobs) + if err != nil { + panic(err) + } + return string(b) +} + func (jobs *Jobs) scan() { jobs.m.Lock() defer jobs.m.Unlock() + debugLogJobs("scan on: '%s'\n", *jobs) running := []*Job{} notrunning := []*Job{} @@ -164,6 +241,7 @@ func (jobs *Jobs) Running() bool { if len(jobs.running) == 0 { return false } + debugLogJobs("running on: '%s'\n", *jobs) return true } @@ -172,6 +250,7 @@ func (jobs *Jobs) GetJobs() []*Job { jobs.scan() jobs.m.Lock() defer jobs.m.Unlock() + debugLogJobs("GetJobs on: '%s'\n", *jobs) return append(jobs.running, jobs.previous...) } @@ -180,6 +259,7 @@ func (jobs *Jobs) GetRunningJobs() []*Job { jobs.scan() jobs.m.Lock() defer jobs.m.Unlock() + debugLogJobs("GetRunningJobs on: '%s'\n", *jobs) return jobs.running } @@ -188,6 +268,7 @@ func (jobs *Jobs) GetStoppedJobs() []*Job { jobs.scan() jobs.m.Lock() defer jobs.m.Unlock() + debugLogJobs("GetStoppedJobs on: '%s'\n", *jobs) return jobs.previous } @@ -195,6 +276,7 @@ func (jobs *Jobs) GetStoppedJobs() []*Job { func (jobs *Jobs) addJob(j *Job) { jobs.m.Lock() jobs.running = append(jobs.running, j) + debugLogJobs("addJob on: '%s'\n", *jobs) jobs.m.Unlock() jobs.scan() } diff --git a/steam/steam.go b/steam/steam.go index 66ef7f4..c477191 100644 --- a/steam/steam.go +++ b/steam/steam.go @@ -11,7 +11,10 @@ import ( "sync" ) -var E_GameDoesNotExist = errors.New("Game does not exist") +var ( + E_GameDoesNotExist = errors.New("Game does not exist") + E_BadURI = errors.New("The URI supplied is not understood") +) // Library is used to represent the steam library, the Games map is populated // by NewLibrary when called or when ProcessLibrary is called directly @@ -69,28 +72,37 @@ func NewLibraryMust(path string) *Library { return l } -// Games returns a slice of *Game for the current library -func (l *Library) Games() []*Game { +// Folder returns the current folder on the disk that contains the steam library +func (l *Library) Folder() string { l.m.Lock() - out := []*Game{} - for _, g := range l.games { - out = append(out, g) - } - l.m.Unlock() + defer l.m.Unlock() + return l.folder +} - return out +// Games returns a map of string[*Game] for the current library +func (l *Library) Games() map[string]*Game { + l.m.Lock() + defer l.m.Unlock() + return l.games } // Jobs returns the current *Jobs struct which can be used to keep track // of any long running operations on the library as well as any errors // encountered along the way -func (l *Library) Status() *Jobs { - return l.status +func (l *Library) Status() Jobs { + l.m.Lock() + defer l.m.Unlock() + return *l.status +} + +// Refresh simply calls ProcessLibrary to refresh the entire contents of the +// steam library. Will return an error if any jobs are running +func (l *Library) Refresh() error { + return l.ProcessLibrary(l.folder) } // ProcessLibrary Populates the "Folder" and "Games" fields based on the -// provided directory. -// +// provided directory. Returns an error if any jobs are currently running func (s *Library) ProcessLibrary(r string) error { if s.status.Running() { return errors.New("Cannot process library with actions running") -- cgit v1.2.3