From 2fcd341fd9c62ca1791f49406590d38966054a16 Mon Sep 17 00:00:00 2001 From: kreativmonkey Date: Sat, 1 Jul 2017 17:58:33 +0200 Subject: [PATCH] Testing and Link disolve Close #8 --- .gitignore | 1 + api.go | 56 ++++++++++++++ config.go | 2 +- handlers.go | 60 +-------------- main.go | 18 +---- routs.go | 4 +- shrty/short.go | 106 ++++++++++++++++++++++++++ shrty/short_test.go | 168 ++++++++++++++++++++++++++++++++++++++++++ shrty/shrty.go | 96 ++---------------------- shrty/shrty_test.go | 17 +++++ shrty/urlinfo.go | 23 ++++++ shrty/urlinfo_test.go | 24 ++++++ 12 files changed, 410 insertions(+), 165 deletions(-) create mode 100644 shrty/short.go create mode 100644 shrty/short_test.go create mode 100644 shrty/shrty_test.go create mode 100644 shrty/urlinfo.go create mode 100644 shrty/urlinfo_test.go diff --git a/.gitignore b/.gitignore index c793993..3459455 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +sgot.db test.db shrt diff --git a/api.go b/api.go index ce07eb2..2ccccdb 100644 --- a/api.go +++ b/api.go @@ -2,6 +2,9 @@ package main import ( "net/http" + "net/url" + "encoding/json" + "github.com/kreativmonkey/shrt/shrty" ) type response struct { @@ -16,3 +19,56 @@ func APIRequest(w http.ResponseWriter, r *http.Request) { } + +// GET: http://example.com/api/action/shorten?key=API_KEY_HERE&url=https://google.com&custom_ending=CUSTOM_ENDING +// Response: {"action": "shorten","result": "https://example.com/5kq"} +func APIshorten(w http.ResponseWriter, r *http.Request){ + query := r.URL.Query() + queryu, err := url.QueryUnescape(query.Get("url")) + check(err) + + var token string + err = short.Short(queryu, &token) + check(err) + + respond := response{ + Action: "shorten", + Result: r.Host + "/" + token, + } + + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(&respond); err != nil { + panic(err) + } + +} + +// GET: http://example.com/api/action/lookup?key=API_KEY_HERE&url_ending=5kq +// Response: {"action":"lookup","result": {"long_url": "https:\/\/google.com","created_at": {"date":"2016-02-12 15:20:34.000000","timezone_type":3,"timezone":"UTC"},"clicks":"0"}} +func APIlookup(w http.ResponseWriter, r *http.Request){ + query := r.URL.Query() + queryu, err := url.QueryUnescape(query.Get("url_ending")) + check(err) + + var result shrty.Data + var respond response + if ok := short.Get(queryu, &result); ok { + respond := response{ + Action: "lookup", + Result: result, + } + + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(&respond); err != nil { + panic(err) + } + } else { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusNotFound) + if err := json.NewEncoder(w).Encode(&respond); err != nil { + panic(err) + } + } +} \ No newline at end of file diff --git a/config.go b/config.go index 65da9dd..956325e 100644 --- a/config.go +++ b/config.go @@ -8,7 +8,7 @@ import ( func DefaultConfig() Config { return Config{ - DB: "./test.db", + DB: "./sgot.db", APIkey: "thisIsNotASecretTokenNow", Host: "localhost", Port: "6889", diff --git a/handlers.go b/handlers.go index 5f0f8de..6cf7147 100644 --- a/handlers.go +++ b/handlers.go @@ -2,11 +2,8 @@ package main import ( "net/http" - "encoding/json" "fmt" "text/template" - "net/url" - "github.com/kreativmonkey/shrt/shrty" ) type Page struct { @@ -26,7 +23,7 @@ func index(w http.ResponseWriter, r *http.Request) { // Redirect to the URL behind the requested token. func redirect(w http.ResponseWriter, r *http.Request){ - if redirect, ok := short.Get(r.URL.Path[1:]); ok{ + if redirect, ok := short.Redirect(r.URL.Path[1:]); ok{ http.Redirect(w, r, string(redirect), 301) fmt.Printf("Token: %s Redirect to: %s \n", string(r.URL.Path[1:]), redirect) } else { @@ -46,62 +43,9 @@ func shorten(w http.ResponseWriter, r *http.Request){ t.Execute(w, P) } -// GET: http://example.com/api/action/shorten?key=API_KEY_HERE&url=https://google.com&custom_ending=CUSTOM_ENDING -// Response: {"action": "shorten","result": "https://example.com/5kq"} -func shortenJSON(w http.ResponseWriter, r *http.Request){ - query := r.URL.Query() - queryu, err := url.QueryUnescape(query.Get("url")) - check(err) - - var token string - err = short.Short(queryu, &token) - check(err) - - respond := response{ - Action: "shorten", - Result: r.Host + "/" + token, - } - - w.Header().Set("Content-Type", "application/json; charset=UTF-8") - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(&respond); err != nil { - panic(err) - } - -} - -// GET: http://example.com/api/action/lookup?key=API_KEY_HERE&url_ending=5kq -// Response: {"action":"lookup","result": {"long_url": "https:\/\/google.com","created_at": {"date":"2016-02-12 15:20:34.000000","timezone_type":3,"timezone":"UTC"},"clicks":"0"}} -func lookupJSON(w http.ResponseWriter, r *http.Request){ - query := r.URL.Query() - queryu, err := url.QueryUnescape(query.Get("url_ending")) - check(err) - - var result shrty.Data - var respond response - if ok := short.GetAPI(queryu, &result); ok { - respond := response{ - Action: "lookup", - Result: result, - } - - w.Header().Set("Content-Type", "application/json; charset=UTF-8") - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(&respond); err != nil { - panic(err) - } - } else { - w.Header().Set("Content-Type", "application/json; charset=UTF-8") - w.WriteHeader(http.StatusNotFound) - if err := json.NewEncoder(w).Encode(&respond); err != nil { - panic(err) - } - } -} - func all(w http.ResponseWriter, r *http.Request){ t, _ := template.New("all").Parse("{{.}}") - t.Execute(w, short.All()) + t.Execute(w, short.GetAll()) } func check(err error) { diff --git a/main.go b/main.go index 08964ac..ab5b759 100644 --- a/main.go +++ b/main.go @@ -8,10 +8,10 @@ import ( "fmt" ) -const version = "0.03" +const version = "0.04" var ( - short *shrty.Storage + short *shrty.Store // Global configuration Variable config = ReadConfig() @@ -23,18 +23,6 @@ func init() { if err != nil { panic(err) } - /* - viper.SetDefault("ContentDir", "content") - viper.SetDefault("Token", map[string]string{"url": "url", "token": "token"}) - - viper.SetConfigName("config") // name of config file (without expression) - viper.AddConfigPath("/etc/appname/") // path to look for the config file in - viper.AddConfigPath("$HOME/.appname") // call multiple times to add many search paths - viper.AddConfigPath(".") // optionally look for config in the working directory - err = viper.ReadInConfig() // Find and read the config file - if err != nil { // Handle errors reading the config file - panic(fmt.Errorf("Fatal error config file: %s \n", err)) - }*/ } type shrt struct { @@ -57,7 +45,7 @@ func main() { http.Handle("/img/", http.StripPrefix("/img/", http.FileServer(http.Dir("template/img")))) */ fmt.Printf("Shrty %s is listen on Port: %s \n", version, config.Port) - fmt.Printf("For the API use following key: %s \n", version, config.APIkey) + fmt.Printf("For the API use following key: %s \n", config.APIkey) http.Handle("/", router) log.Fatal(http.ListenAndServe(":"+ config.Port, router)) diff --git a/routs.go b/routs.go index 437c67f..d36ec77 100644 --- a/routs.go +++ b/routs.go @@ -40,13 +40,13 @@ var routes = Routes{ "ShortenAPI", "GET", "/api/v1/action/shorten", - shortenJSON, + APIshorten, }, Route{ "LookupAPI", "GET", "/api/v1/action/lookup", - lookupJSON, + APIlookup, }, Route{ "GetAll", diff --git a/shrty/short.go b/shrty/short.go new file mode 100644 index 0000000..6aec8e8 --- /dev/null +++ b/shrty/short.go @@ -0,0 +1,106 @@ +package shrty + +import ( + "fmt" + "crypto/sha256" + "time" + "encoding/json" +) + +type Data struct { + URL string `json:"url"` + URLFetched string `json:"url_fetched"` + CanonicalURL string `json:"canonical_url"` + OriginalURL string `json:"original_url"` + Domain string `json:"domain"` + Hash string `json:"hash"` + Token string `json:"token"` + Meta `json:"meta"` + FavIconLink string `json:"favicon_url"` + HTTPStatusCode string `json:"http_code"` + Category string `json:"category"` + Created string `json:"created_at"` + Clicks int64 `json:"clicks"` +} + +type Meta struct{ + Title string `json:"title"meta:"og:title"` + Image string `json:"image"meta:"og:image"` + Description string `json:"description"meta:"og:description"` + Type string `json:"type"meta:"og:type"` +} + +func (s *Store) Short(URL string, value *string) error { + d := &Data{ + URL: URL, + } + + err := CheckURL(d) + if err != nil { + return err + } + + // Create a sha256 Hash from the URL + d.Hash = fmt.Sprintf("%x", sha256.Sum256([]byte(d.OriginalURL))) + + // Check if the URL already in the Storage + if ok := s.Exist(d); ok { + *value = d.Token + return nil + } + + // Iterate to the length of the hash to get the shortest output + for hashShortestLen := 1; hashShortestLen <= 32; hashShortestLen++ { + // Test if the Token not exist and return the new generated token + if _, ok := s.Token[d.Hash[:hashShortestLen]]; !ok { + d.Token = d.Hash[:hashShortestLen] + d.Created = time.Now().String() + + s.Token[d.Token] = d + s.Url[d.Hash] = d.Token + *value = d.Token + + s.Save() + return nil + } + } + + return ErrCreateToken +} + +// URL already Exist in the +func (s *Store) Exist(d *Data) bool { + if val, ok := s.Url[d.Hash]; ok { + d = s.Token[val] + return true + } + return false +} + +// Returns the URL for the given token +func (s *Store) Redirect(token string) (string, bool) { + if shrt, ok := s.Token[token]; ok { + s.Token[token].Clicks += 1 + s.Save() + return shrt.OriginalURL, true + } + return "", false +} + +// Get Data for the given Token +func (s *Store) Get(token string, value *Data) bool { + if data, ok := s.Token[token]; ok { + *value = *data + return true + } + return false +} + +// Get all entries +func (s *Store) GetAll() string { + b, err := json.Marshal(&s) + if err != nil { + return "" + } + return string(b) +} \ No newline at end of file diff --git a/shrty/short_test.go b/shrty/short_test.go new file mode 100644 index 0000000..76a3972 --- /dev/null +++ b/shrty/short_test.go @@ -0,0 +1,168 @@ +package shrty + +import ( + "github.com/stretchr/testify/assert" + "testing" + "os" + "time" +) + +const path = "short_test.db" + +// This test seems to be not working correctly +func TestStorage_Short(t *testing.T) { + s, err := Open(path) + defer os.Remove(path) + + assert.NoError(t, err) + + var token string + + err = s.Short("http://bit.ly/2scBYES", &token) + + expected := &Data{ + URL:"http://bit.ly/2scBYES", + URLFetched:"https://www.youtube.com/watch?v=bouIpFd9VGM", + CanonicalURL:"", + OriginalURL:"https://www.youtube.com/watch?v=bouIpFd9VGM", + Domain:"www.youtube.com", + Hash:"c921ca733b92ca7b57782e9f12a3fe60dbc6b91a317fc419e13f8f5b5805232f", + Token:"c", + Meta: Meta{ + Title:"", + Image:"", + Description:"", + Type:""}, + FavIconLink:"", + HTTPStatusCode:"200", + Category:"", + Created:s.Token[token].Created, + Clicks:0, + } + + assert.NoError(t, err) + assert.Equal(t, expected, s.Token[token]) +} + +// Whats wrong with this Test? +func TestStorage_Get(t *testing.T) { + s, err := Open(path) + defer os.Remove(path) + + var d Data + empty := Data{} + + ok := s.Get("notexist", &d) + + assert.Equal(t, empty, d) + assert.Equal(t, false, ok) + + var token string + err = s.Short("http://bit.ly/2scBYES", &token) + assert.NoError(t, err) + assert.Equal(t, "c", token) + + expected := &Data{ + URL:"http://bit.ly/2scBYES", + URLFetched:"https://www.youtube.com/watch?v=bouIpFd9VGM", + CanonicalURL:"", + OriginalURL:"https://www.youtube.com/watch?v=bouIpFd9VGM", + Domain:"www.youtube.com", + Hash:"c921ca733b92ca7b57782e9f12a3fe60dbc6b91a317fc419e13f8f5b5805232f", + Token:"c", + Meta: Meta{ + Title:"", + Image:"", + Description:"", + Type:""}, + FavIconLink:"", + HTTPStatusCode:"200", + Category:"", + Created:s.Token[token].Created, + Clicks:0, + } + + ok = s.Get(token, &d) + + assert.Equal(t, expected, &d) +} + +func TestStore_Exist(t *testing.T) { + s, err := Open("test.db") + defer os.Remove(path) + + d := &Data{ + URL:"http://bit.ly/2scBYES", + URLFetched:"https://www.youtube.com/watch?v=bouIpFd9VGM", + CanonicalURL:"", + OriginalURL:"https://www.youtube.com/watch?v=bouIpFd9VGM", + Domain:"www.youtube.com", + Hash:"c921ca733b92ca7b57782e9f12a3fe60dbc6b91a317fc419e13f8f5b5805232f", + Token:"c", + Meta: Meta{ + Title:"", + Image:"", + Description:"", + Type:""}, + FavIconLink:"", + HTTPStatusCode:"200", + Category:"", + Created:time.Now().String(), + Clicks:0, + } + + notexist := s.Exist(d) + assert.Equal(t, false, notexist) + + var token string + + err = s.Short(d.URL, &token) + assert.NoError(t, err) + + exist := s.Exist(d) + + assert.Equal(t, true, exist) +} + +func TestStore_Redirect(t *testing.T) { + s, err := Open(path) + defer os.Remove(path) + + url, ok := s.Redirect("notexist") + + assert.Equal(t, "", url) + assert.Equal(t, false, ok) + + var token string + err = s.Short("http://bit.ly/2scBYES", &token) + + assert.NoError(t, err) + + expected := &Data{ + URL:"http://bit.ly/2scBYES", + URLFetched:"https://www.youtube.com/watch?v=bouIpFd9VGM", + CanonicalURL:"", + OriginalURL:"https://www.youtube.com/watch?v=bouIpFd9VGM", + Domain:"www.youtube.com", + Hash:"c921ca733b92ca7b57782e9f12a3fe60dbc6b91a317fc419e13f8f5b5805232f", + Token:"c", + Meta: Meta{ + Title:"", + Image:"", + Description:"", + Type:""}, + FavIconLink:"", + HTTPStatusCode:"200", + Category:"", + Created:time.Now().String(), + Clicks:0, + } + + url, ok = s.Redirect(token) + + assert.Equal(t, expected.OriginalURL, url) +} + +//func TestStore_GetAll(t *testing.T) { + +//} \ No newline at end of file diff --git a/shrty/shrty.go b/shrty/shrty.go index c0ab890..6e0b9b1 100644 --- a/shrty/shrty.go +++ b/shrty/shrty.go @@ -2,32 +2,15 @@ package shrty import ( "bytes" - "crypto/sha256" "encoding/json" "errors" - "fmt" "io" - "net/url" "os" - "time" ) -type Storage struct { - Token map[string]*Data `json:"token"` - Url map[string]string `json:"url"` -} - -type Data struct { - URL string `json:"url"` - URLFetched string `json:"url_fetched"` - CanonicalURL string `json:"canonical_url"` - OriginalURL string `json:"original_url"` - Domain string `json:"domain"` - FavIconLink string `json:"favicon_url"` - HTTPStatusCode string `json:"http_code"` - Category string `json:"category"` - Created string `json:"created_at"` - Clicks int64 `json:"clicks"` +type Store struct { + Token map[string]*Data `json:"token"` // [token]*Data + Url map[string]string `json:"url"` // [hash]token } var ( @@ -37,8 +20,8 @@ var ( ) // Open up -func Open(path string) (*Storage, error) { - s := Storage{Token: make(map[string]*Data), Url: make(map[string]string)} +func Open(path string) (*Store, error) { + s := Store{Token: make(map[string]*Data), Url: make(map[string]string)} // Open db ore create if not exist! if db, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644); err == nil { json.Unmarshal(StreamToByte(db), &s) @@ -49,76 +32,11 @@ func Open(path string) (*Storage, error) { } } -func (s *Storage) Short(URL string, value *string) error { - // Check if it is a valide Url found on: - // http://stackoverflow.com/questions/31480710/validate-url-with-standard-package-in-go - _, err := url.ParseRequestURI(URL) - if err != nil { - return ErrNoUrl - } - - // Create a sha256 Hash from the URL - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(URL))) - - // Test if the URL alraedy exist and return the key - if val, ok := s.Url[hash]; ok { - *value = val - return nil - } - // Iterate to the length of hash to get the shortest output - for hashShortestLen := 1; hashShortestLen <= 32; hashShortestLen++ { - // Test if the Token not exist and return the new generated token - if _, ok := s.Token[hash[:hashShortestLen]]; !ok { - token := hash[:hashShortestLen] - t := time.Now() - s.Token[token] = &Data{ - URL: URL, - Created: t.String(), - } - s.Url[hash] = token - *value = s.Url[hash] - s.Save() - return nil - } - } - - return ErrCreateToken -} - -func (s *Storage) Remove(URL string) error { +func (s *Store) Remove(URL string) error { return nil } -// Get returns the URL for the given token -func (s *Storage) Get(token string) (string, bool) { - if shrt, ok := s.Token[token]; ok { - fmt.Printf("Url mit dem Token %s gefunden: %s \n", token, shrt.URL) - s.Token[token].Clicks += 1 - s.Save() - return shrt.URL, true - } - return "", false -} - -// Get returns the URL for the given token -func (s *Storage) GetAPI(token string, value *Data) bool { - if shrt, ok := s.Token[token]; ok { - *value = *shrt - return true - } - return false -} - -// Get all entries -func (s *Storage) All() string { - b, err := json.Marshal(&s) - if err != nil { - return "" - } - return string(b) -} - -func (s *Storage) Save() error { +func (s *Store) Save() error { if db, err := os.OpenFile("./test.db", os.O_RDWR|os.O_CREATE, 0644); err == nil { b, err := json.Marshal(&s) db.Write(b) diff --git a/shrty/shrty_test.go b/shrty/shrty_test.go new file mode 100644 index 0000000..8bcf5b0 --- /dev/null +++ b/shrty/shrty_test.go @@ -0,0 +1,17 @@ +package shrty + +import ( + "os" + "testing" + "github.com/stretchr/testify/assert" +) + +func TestOpen(t *testing.T) { + s, err := Open(path) + defer os.Remove(path) + + expected := Store{Token: make(map[string]*Data), Url: make(map[string]string)} + + assert.NoError(t, err) + assert.Equal(t, &expected, s) +} diff --git a/shrty/urlinfo.go b/shrty/urlinfo.go new file mode 100644 index 0000000..2509088 --- /dev/null +++ b/shrty/urlinfo.go @@ -0,0 +1,23 @@ +package shrty + +import ( + "fmt" + "net/http" +) + + +func CheckURL(d *Data) error { + resp, err := http.Get(d.URL) + if err != nil { + fmt.Println(err) + } + defer resp.Body.Close() + + + d.HTTPStatusCode = fmt.Sprint(resp.StatusCode) + d.URLFetched = fmt.Sprint(resp.Request.URL) + d.OriginalURL = fmt.Sprint(resp.Request.URL) + d.Domain = resp.Request.URL.Host + + return err +} diff --git a/shrty/urlinfo_test.go b/shrty/urlinfo_test.go new file mode 100644 index 0000000..59bbd55 --- /dev/null +++ b/shrty/urlinfo_test.go @@ -0,0 +1,24 @@ +package shrty + +import ( + "testing" + "github.com/stretchr/testify/assert" +) + +func TestCheckURL(t *testing.T) { + data := Data{ + URL: "https://t.co/kA45uWnKkg", + } + + expected := Data{ + URL: "https://t.co/kA45uWnKkg", + URLFetched: "https://www.youtube.com/watch?v=Qg0pO9VG1J8&utm_content=buffera4269&utm_medium=social&utm_source=twitter.com&utm_campaign=buffer", + OriginalURL: "https://www.youtube.com/watch?v=Qg0pO9VG1J8&utm_content=buffera4269&utm_medium=social&utm_source=twitter.com&utm_campaign=buffer", + HTTPStatusCode: "200", + Domain: "www.youtube.com", + } + + err := CheckURL(&data) + assert.NoError(t, err) + assert.Equal(t, expected, data) +}