aboutsummaryrefslogtreecommitdiff
path: root/steam
diff options
context:
space:
mode:
authorMitch Riedstra <mitch@riedstra.us>2021-08-04 23:53:36 -0400
committerMitch Riedstra <mitch@riedstra.us>2021-08-04 23:53:36 -0400
commitc202f2eca32e1ab2e313417168351df1c58ee062 (patch)
tree6540629b337d2d769581baec26096ac0555f71f9 /steam
parent742938b00222c7ad57ad11eb24850d9202c2503d (diff)
downloadsteam-export-c202f2eca32e1ab2e313417168351df1c58ee062.tar.gz
steam-export-c202f2eca32e1ab2e313417168351df1c58ee062.tar.xz
More major changes. Web UI works. Downloading games works. Status works. extractFile needs work
Diffstat (limited to 'steam')
-rw-r--r--steam/delete.go41
-rw-r--r--steam/extract.go154
-rw-r--r--steam/extract_http.go (renamed from steam/http.go)27
-rw-r--r--steam/game.go9
-rw-r--r--steam/package.go181
-rw-r--r--steam/status.go82
-rw-r--r--steam/steam.go38
7 files changed, 367 insertions, 165 deletions
diff --git a/steam/delete.go b/steam/delete.go
new file mode 100644
index 0000000..486b047
--- /dev/null
+++ b/steam/delete.go
@@ -0,0 +1,41 @@
+package steam
+
+import (
+ "os"
+ "path/filepath"
+)
+
+// Delete removes all of the game files and the ACF
+func (l *Library) Delete(game string) error {
+ g, ok := l.games[game]
+ if !ok {
+ return E_GameDoesNotExist
+ }
+
+ j := newJob("delete", g)
+ defer j.done()
+
+ l.status.addJob(j)
+
+ acf, err := FindACF(l.folder, game)
+ if err != nil {
+ j.addError(err)
+ return err
+ }
+ if err := os.Remove(acf); err != nil {
+ j.addError(err)
+ return err
+ }
+
+ err = os.RemoveAll(filepath.Join(l.folder, "common", g.Name))
+ if err != nil {
+ j.addError(err)
+ return err
+ }
+
+ l.m.Lock()
+ delete(l.games, game)
+ l.m.Unlock()
+
+ return nil
+}
diff --git a/steam/extract.go b/steam/extract.go
index f73d5f7..9a7a930 100644
--- a/steam/extract.go
+++ b/steam/extract.go
@@ -1,15 +1,90 @@
package steam
import (
+ "archive/tar"
"errors"
"fmt"
"io"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
"time"
)
// how often are we going to be updating our status information?
const updateEveryNBytes = 10 * 1024 * 1024 // 10mb
+// ExtractSmart attempts to discover what kind of resource is behind uri
+// and extract it appropriately. It may fail with E_BadURI.
+//
+// For example the following forms are accepted:
+//
+// ExtractSmart("http://127.0.0.1/some-archive")
+// ExtractSmart("https://example.com/some-archive")
+// ExtractSmart("file:///some/local/file/path/to/archive.tar")
+// ExtractSmart("/direct/path/to/archive.tar")
+// ExtractSmart("C:\Users\user\Downloads\archive.tar")
+func (l *Library) ExtractSmart(uri string) (*Game, error) {
+ if strings.HasPrefix(uri, "http") {
+ _, err := url.Parse(uri)
+ if err == nil {
+ return l.ExtractHTTP(uri)
+ }
+ } else if strings.HasPrefix(uri, "file") {
+ u, err := url.Parse(uri)
+ if err == nil {
+ return l.ExtractFile(u.Path)
+ }
+ } else if _, err := os.Stat(uri); err == nil {
+ return l.ExtractFile(uri)
+ }
+
+ return nil, E_BadURI
+}
+
+// ExtractFile is a wrapper around Extract that handles an HTTP endpoint.
+// this spawns an "extractFile" on the library. Status will be updated there
+// as this goes along. Non fatal and fatal errors will be populated there
+func (l *Library) ExtractFile(fn string) (*Game, error) {
+ g := &Game{}
+ j := newJob("extractFile", g)
+ defer j.done()
+
+ l.status.addJob(j)
+
+ fi, err := os.Stat(fn)
+ if err != nil {
+ j.addError(err)
+ return g, err
+ }
+ j.setSize(fi.Size())
+
+ fh, err := os.Open(fn)
+ if err != nil {
+ j.addError(err)
+ return g, err
+ }
+
+ return l.extractUpdate(j, g, fh)
+}
+
+// Extract will read a tarball from the io.Reader and install the game into
+// the current library path. This offers no visibility into the progress,
+// as it does not update the job status on the progress, though it will
+// populate errors.
+//
+// Most callers will want to use ExtractHTTP or ExtractFile instead
+func (l *Library) Extract(r io.Reader) (*Game, error) {
+ g := &Game{LibraryPath: l.folder}
+ j := newJob("extract", g)
+ defer j.done()
+
+ l.status.addJob(j)
+
+ return l.extractPrimitive(j, g, r)
+}
+
// extractUpdate takes care of updating the job as it goes along at updateEveryNBytes
// it will be reported back to the Job's status.
func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, error) {
@@ -43,7 +118,8 @@ func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, erro
j.setTransferred(total)
// rate in bytes/sec
- rate := total / int64(time.Since(*j.StartTime()).Seconds())
+ // rate := total / int64(time.Since(*j.StartTime()).Seconds())
+ rate := float64(total) / float64(time.Since(*j.StartTime()).Seconds())
estSize := j.GetSize()
@@ -52,7 +128,8 @@ func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, erro
continue
}
- remaining := *estSize - total
+ // remaining := *estSize - total
+ remaining := float64(*estSize - total)
j.setETA(time.Duration((remaining / rate) / 1000 / 1000 / 1000))
}
@@ -64,3 +141,76 @@ func (l *Library) extractUpdate(j *Job, g *Game, rdr io.ReadCloser) (*Game, erro
return g, err
}
+
+func (l *Library) extractPrimitive(j *Job, g *Game, r io.Reader) (*Game, error) {
+ treader := tar.NewReader(r)
+
+ for {
+ hdr, err := treader.Next()
+ if err == io.EOF {
+ // We've reached the end! Whoee
+ break
+ }
+ if err != nil {
+ j.addError(err)
+ return nil, err
+ }
+
+ fileName := filepath.ToSlash(hdr.Name)
+
+ if g.Name == "" {
+ s := strings.Split(fileName, "/")
+ if len(s) >= 2 {
+ g.Name = s[1]
+ }
+ }
+
+ fileName = filepath.Join(l.folder, fileName)
+
+ info := hdr.FileInfo()
+ if info.IsDir() {
+ // I don't like hard-coded permissions but it
+ // it helps with overall platform compatibility
+ err = os.MkdirAll(fileName, defaultDirectoryMode)
+ if err != nil {
+ j.addError(err)
+ return nil, err
+ }
+
+ continue
+ }
+
+ err = os.MkdirAll(filepath.Dir(fileName), defaultDirectoryMode)
+ if err != nil {
+ j.addError(err)
+ return nil, err
+ }
+
+ // Create a file handle to work with
+ f, err := os.OpenFile(fileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
+ defaultFileMode)
+ if err != nil {
+ j.addError(err)
+ return nil, err
+ }
+ if _, err := io.Copy(f, treader); err != nil {
+ j.addError(err)
+ f.Close()
+ return nil, err
+ }
+ f.Close()
+
+ }
+
+ err := g.SetSizeInfo()
+ if err != nil {
+ j.addError(err)
+ return nil, err
+ }
+
+ l.m.Lock()
+ l.games[g.Name] = g
+ l.m.Unlock()
+
+ return g, nil
+}
diff --git a/steam/http.go b/steam/extract_http.go
index df756e4..6884bee 100644
--- a/steam/http.go
+++ b/steam/extract_http.go
@@ -3,36 +3,9 @@ package steam
import (
"fmt"
"net/http"
- "os"
"strconv"
)
-// ExtractFile is a wrapper around Extract that handles an HTTP endpoint.
-// this spawns an "extractFile" on the library. Status will be updated there
-// as this goes along. Non fatal and fatal errors will be populated there
-func (l *Library) ExtractFile(fn string) (*Game, error) {
- g := &Game{}
- j := newJob("extractFile", g)
- defer j.done()
-
- l.status.addJob(j)
-
- fi, err := os.Stat(fn)
- if err != nil {
- j.addError(err)
- return g, err
- }
- j.setSize(fi.Size())
-
- fh, err := os.Open(fn)
- if err != nil {
- j.addError(err)
- return g, err
- }
-
- return l.extractUpdate(j, g, fh)
-}
-
// ExtractHTTP is a wrapper around Extract that handles an HTTP endpoint.
// this spawns an "extractHTTP" on the library. Status will be updated there
// as this goes along. Non fatal and fatal errors will be populated there
diff --git a/steam/game.go b/steam/game.go
index 0028257..dd4a297 100644
--- a/steam/game.go
+++ b/steam/game.go
@@ -5,6 +5,15 @@ import (
"path/filepath"
)
+// GetSizeBytes returns the size in bytes, calling SetSizeInfo info Size is
+// currently == 0
+func (g *Game) GetSizeBytes() int64 {
+ if g.Size == 0 {
+ _ = g.SetSizeInfo()
+ }
+ return g.Size
+}
+
// GetSize returns the size of a game in a pretty format. If size is 0
// it will call SetSizeInfo before returning
func (g *Game) GetSize() string {
diff --git a/steam/package.go b/steam/package.go
index df6f1e8..5b0c618 100644
--- a/steam/package.go
+++ b/steam/package.go
@@ -2,10 +2,11 @@ package steam
import (
"archive/tar"
+ "errors"
+ "fmt"
"io"
- "os"
"path/filepath"
- "strings"
+ "time"
)
const (
@@ -13,177 +14,111 @@ const (
defaultFileMode = 0644
)
-// Package writes the package to wr, returning errors if any
-func (l *Library) Package(game string, wr io.WriteCloser) error {
- g, ok := l.games[game]
- if !ok {
+func (l *Library) Package(game string, wr io.Writer) error {
+ if _, ok := l.Games()[game]; !ok {
return E_GameDoesNotExist
}
+ g := l.Games()[game]
j := newJob("package", g)
defer j.done()
l.status.addJob(j)
- acf, err := FindACF(l.folder, g.Name)
- if err != nil {
- j.addError(err)
- return err
- }
+ j.setSize(g.GetSizeBytes())
- twriter := tar.NewWriter(wr)
-
- paths := []string{
- filepath.Join(l.folder, "common", g.Name),
- acf,
- }
- for _, pth := range paths {
- err := filepath.Walk(pth, tarWalkfn(twriter, l.folder))
+ // Invert the writer so we can break up the copy and get progress
+ // information in here
+ rdr, pwrtr := io.Pipe()
+ go func() {
+ err := l.packagePrimitive(j, g, pwrtr)
if err != nil {
j.addError(err)
- return err
}
- }
-
- err = twriter.Flush()
- if err != nil {
- j.addError(err)
- return err
- }
-
- err = twriter.Close()
- if err != nil {
- j.addError(err)
- return err
- }
-
- err = wr.Close()
- if err != nil {
- j.addError(err)
- }
- return err
-}
-
-// Extract will read a tarball from the io.Reader and install the game into
-// the current library path. This offers no visibility into the progress,
-// as it does not update the job status on the progress, though it will
-// populate errors.
-//
-// Most callers will want to use ExtractHTTP or ExtractFile instead
-func (l *Library) Extract(r io.Reader) (*Game, error) {
- g := &Game{LibraryPath: l.folder}
- j := newJob("extract", g)
- defer j.done()
-
- l.status.addJob(j)
-
- return l.extractPrimitive(j, g, r)
-}
-
-func (l *Library) extractPrimitive(j *Job, g *Game, r io.Reader) (*Game, error) {
- treader := tar.NewReader(r)
+ }()
+ var total int64
for {
- hdr, err := treader.Next()
+ n, err := io.CopyN(wr, rdr, updateEveryNBytes)
+
if err == io.EOF {
- // We've reached the end! Whoee
break
}
if err != nil {
j.addError(err)
- return nil, err
+ return err
}
- fileName := filepath.ToSlash(hdr.Name)
+ total += n
+ j.setTransferred(total)
- if g.Name == "" {
- s := strings.Split(fileName, "/")
- if len(s) >= 2 {
- g.Name = s[1]
- }
- }
+ elapsedSeconds := float64(time.Since(*j.StartTime()).Seconds())
+
+ rate := float64(total) / elapsedSeconds
- fileName = filepath.Join(l.folder, fileName)
+ fmt.Println("rate in bytes/second: ", formatBytes(int64(rate)))
- info := hdr.FileInfo()
- if info.IsDir() {
- // I don't like hard-coded permissions but it
- // it helps with overall platform compatibility
- err = os.MkdirAll(fileName, defaultDirectoryMode)
- if err != nil {
- j.addError(err)
- return nil, err
- }
+ estSize := j.GetSize()
+ if estSize == nil {
+ j.addError(errors.New("Expected an estimated size, got nil"))
continue
}
- err = os.MkdirAll(filepath.Dir(fileName), defaultDirectoryMode)
- if err != nil {
- j.addError(err)
- return nil, err
- }
+ remainingBytes := float64(*estSize - total)
+ fmt.Println("remaining bytes: ", formatBytes(int64(remainingBytes)))
- // Create a file handle to work with
- f, err := os.OpenFile(fileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
- defaultFileMode)
- if err != nil {
- j.addError(err)
- return nil, err
- }
- if _, err := io.Copy(f, treader); err != nil {
- j.addError(err)
- f.Close()
- return nil, err
- }
- f.Close()
+ seconds := (remainingBytes / rate)
- }
+ duration := time.Duration(seconds * 1000 * 1000 * 1000)
+ fmt.Println("Raw duration: ", duration)
- err := g.SetSizeInfo()
- if err != nil {
- j.addError(err)
- return nil, err
+ j.setETA(duration)
}
- l.m.Lock()
- l.games[g.Name] = g
- l.m.Unlock()
-
- return g, nil
+ return nil
}
-// Delete removes all of the game files and the ACF
-func (l *Library) Delete(game string) error {
- g, ok := l.games[game]
- if !ok {
- return E_GameDoesNotExist
+// Package writes the package to wr, returning errors if any
+func (l *Library) packagePrimitive(j *Job, g *Game, wr io.WriteCloser) error {
+
+ acf, err := FindACF(l.folder, g.Name)
+ if err != nil {
+ j.addError(err)
+ return err
}
- j := newJob("delete", g)
- defer j.done()
+ twriter := tar.NewWriter(wr)
- l.status.addJob(j)
+ paths := []string{
+ filepath.Join(l.folder, "common", g.Name),
+ acf,
+ }
+ for _, pth := range paths {
+ err := filepath.Walk(pth, tarWalkfn(twriter, l.folder))
+ if err != nil {
+ j.addError(err)
+ return err
+ }
+ }
- acf, err := FindACF(l.folder, game)
+ err = twriter.Flush()
if err != nil {
j.addError(err)
return err
}
- if err := os.Remove(acf); err != nil {
+
+ err = twriter.Close()
+ if err != nil {
j.addError(err)
return err
}
- err = os.RemoveAll(filepath.Join(l.folder, "common", g.Name))
+ err = wr.Close()
if err != nil {
j.addError(err)
return err
}
- l.m.Lock()
- delete(l.games, game)
- l.m.Unlock()
-
- return nil
+ return err
}
diff --git a/steam/status.go b/steam/status.go
index 12f5e3b..9d70655 100644
--- a/steam/status.go
+++ b/steam/status.go
@@ -1,10 +1,28 @@
package steam
import (
+ "encoding/json"
"sync"
"time"
+
+ "fmt"
+ "os"
)
+var debuglogging = false
+
+func debugLogJob(s string, args ...interface{}) {
+ if debuglogging {
+ fmt.Fprintf(os.Stderr, s, args...)
+ }
+}
+
+func debugLogJobs(s string, args ...interface{}) {
+ if debuglogging {
+ fmt.Fprintf(os.Stderr, s, args...)
+ }
+}
+
// JobStatus provides specific information about an individual job
type Job struct {
action string
@@ -21,10 +39,36 @@ type Job struct {
m sync.Mutex
}
+func (j Job) MarshalJSON() ([]byte, error) {
+ return json.Marshal(
+ struct {
+ Action string `json:"action"`
+ Target *Game `json:"Target"`
+ Running bool `json:"Running"`
+ Start *time.Time `json:"Start"`
+ Errors []error `json:"Errors"`
+
+ // If applicablle
+ Size *int64 `json:"Size"`
+ Transferred *int64 `json:"Transferred"`
+ Eta *time.Duration `json:"ETA"`
+ }{
+ Action: j.action,
+ Target: j.target,
+ Running: j.running,
+ Start: j.start,
+ Errors: j.errors,
+ Size: j.size,
+ Transferred: j.transferred,
+ Eta: j.eta,
+ })
+}
+
// Action is a short string describing the action, i.e. "packaging", "deleting"
func (j *Job) Action() string {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("Action on: '%s'\n", *j)
return j.action
}
@@ -32,6 +76,7 @@ func (j *Job) Action() string {
func (j *Job) Target() *Game {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("Target on: '%s'\n", *j)
return j.target
}
@@ -39,6 +84,7 @@ func (j *Job) Target() *Game {
func (j *Job) IsRunning() bool {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("IsRunning on: '%s'\n", *j)
return j.running
}
@@ -46,17 +92,20 @@ func (j *Job) IsRunning() bool {
func (j *Job) StartTime() *time.Time {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("StartTime on: '%s'\n", *j)
return j.start
}
func (j *Job) Errors() []error {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("errors on: '%s'\n", *j)
return j.errors
}
// newJob sets up a job of action for the target Game
func newJob(action string, target *Game) *Job {
+ debugLogJob("New job: '%s' target: '%s'\n", action, target)
t := time.Now()
return &Job{
action: action,
@@ -69,6 +118,7 @@ func newJob(action string, target *Game) *Job {
func (j *Job) setSize(size int64) {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("setSize on: '%s'\n", *j)
j.size = &size
}
@@ -76,12 +126,14 @@ func (j *Job) setSize(size int64) {
func (j *Job) GetSize() *int64 {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("getSize on: '%s'\n", *j)
return j.size
}
func (j *Job) setTransferred(transferred int64) {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("setTransferred on: '%s'\n", *j)
j.transferred = &transferred
}
@@ -89,6 +141,7 @@ func (j *Job) setTransferred(transferred int64) {
func (j *Job) GetTransferred() *int64 {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("GetTransferred on: '%s'\n", *j)
return j.transferred
}
@@ -96,6 +149,7 @@ func (j *Job) GetTransferred() *int64 {
func (j *Job) setETA(d time.Duration) {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("setETA on: '%s'\n", *j)
j.eta = &d
}
@@ -104,6 +158,7 @@ func (j *Job) setETA(d time.Duration) {
func (j *Job) GetETA() *time.Duration {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("GetETA on: '%s'\n", *j)
return j.eta
}
@@ -118,6 +173,7 @@ func (j *Job) done() {
func (j *Job) addError(err error) {
j.m.Lock()
defer j.m.Unlock()
+ debugLogJob("add error on: '%s'\n", *j)
j.errors = append(j.errors, err)
}
@@ -131,9 +187,30 @@ type Jobs struct {
m sync.Mutex
}
+func (jobs Jobs) MarshalJSON() ([]byte, error) {
+ jobs.scan()
+ return json.Marshal(
+ struct {
+ Running []*Job `json:"Running"`
+ Previous []*Job `json:"Previous"`
+ }{
+ Running: jobs.running,
+ Previous: jobs.previous,
+ })
+}
+
+func (jobs Jobs) String() string {
+ b, err := json.Marshal(jobs)
+ if err != nil {
+ panic(err)
+ }
+ return string(b)
+}
+
func (jobs *Jobs) scan() {
jobs.m.Lock()
defer jobs.m.Unlock()
+ debugLogJobs("scan on: '%s'\n", *jobs)
running := []*Job{}
notrunning := []*Job{}
@@ -164,6 +241,7 @@ func (jobs *Jobs) Running() bool {
if len(jobs.running) == 0 {
return false
}
+ debugLogJobs("running on: '%s'\n", *jobs)
return true
}
@@ -172,6 +250,7 @@ func (jobs *Jobs) GetJobs() []*Job {
jobs.scan()
jobs.m.Lock()
defer jobs.m.Unlock()
+ debugLogJobs("GetJobs on: '%s'\n", *jobs)
return append(jobs.running, jobs.previous...)
}
@@ -180,6 +259,7 @@ func (jobs *Jobs) GetRunningJobs() []*Job {
jobs.scan()
jobs.m.Lock()
defer jobs.m.Unlock()
+ debugLogJobs("GetRunningJobs on: '%s'\n", *jobs)
return jobs.running
}
@@ -188,6 +268,7 @@ func (jobs *Jobs) GetStoppedJobs() []*Job {
jobs.scan()
jobs.m.Lock()
defer jobs.m.Unlock()
+ debugLogJobs("GetStoppedJobs on: '%s'\n", *jobs)
return jobs.previous
}
@@ -195,6 +276,7 @@ func (jobs *Jobs) GetStoppedJobs() []*Job {
func (jobs *Jobs) addJob(j *Job) {
jobs.m.Lock()
jobs.running = append(jobs.running, j)
+ debugLogJobs("addJob on: '%s'\n", *jobs)
jobs.m.Unlock()
jobs.scan()
}
diff --git a/steam/steam.go b/steam/steam.go
index 66ef7f4..c477191 100644
--- a/steam/steam.go
+++ b/steam/steam.go
@@ -11,7 +11,10 @@ import (
"sync"
)
-var E_GameDoesNotExist = errors.New("Game does not exist")
+var (
+ E_GameDoesNotExist = errors.New("Game does not exist")
+ E_BadURI = errors.New("The URI supplied is not understood")
+)
// Library is used to represent the steam library, the Games map is populated
// by NewLibrary when called or when ProcessLibrary is called directly
@@ -69,28 +72,37 @@ func NewLibraryMust(path string) *Library {
return l
}
-// Games returns a slice of *Game for the current library
-func (l *Library) Games() []*Game {
+// Folder returns the current folder on the disk that contains the steam library
+func (l *Library) Folder() string {
l.m.Lock()
- out := []*Game{}
- for _, g := range l.games {
- out = append(out, g)
- }
- l.m.Unlock()
+ defer l.m.Unlock()
+ return l.folder
+}
- return out
+// Games returns a map of string[*Game] for the current library
+func (l *Library) Games() map[string]*Game {
+ l.m.Lock()
+ defer l.m.Unlock()
+ return l.games
}
// Jobs returns the current *Jobs struct which can be used to keep track
// of any long running operations on the library as well as any errors
// encountered along the way
-func (l *Library) Status() *Jobs {
- return l.status
+func (l *Library) Status() Jobs {
+ l.m.Lock()
+ defer l.m.Unlock()
+ return *l.status
+}
+
+// Refresh simply calls ProcessLibrary to refresh the entire contents of the
+// steam library. Will return an error if any jobs are running
+func (l *Library) Refresh() error {
+ return l.ProcessLibrary(l.folder)
}
// ProcessLibrary Populates the "Folder" and "Games" fields based on the
-// provided directory.
-//
+// provided directory. Returns an error if any jobs are currently running
func (s *Library) ProcessLibrary(r string) error {
if s.status.Running() {
return errors.New("Cannot process library with actions running")