cache page data

models/Movies.go

@@ -23,3 +23,7 @@ func FindMoviesStringValue(key string) string {
 func FindMoviesHashValue(key string) map[string]string {
 	return utils.RedisDB.HGetAll(key).Val()
 }
+
+func SaveMovies(key string, value string) error {
+	return utils.RedisDB.Set(key, value, 0).Err()
+}

services/MoviesService.go

@@ -5,6 +5,7 @@ import (
 	"go_movies/utils"
 	"log"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -98,6 +99,18 @@ func MovieListsRange(key string, start, stop int64) []MovieListStruct {
 
 	var movieKeyMap MovieListStruct
 
+	sStart := strconv.FormatInt(start, 10)
+	sStop := strconv.FormatInt(stop, 10)
+
+	cacheKey := "movie_lists_key:" + key + ":start:" + sStart + ":stop:" + sStop
+
+	movieList := models.FindMoviesStringValue(cacheKey)
+
+	if movieList != "" {
+		utils.Json.Unmarshal([]byte(movieList), &data)
+		return data
+	}
+
 	movieKeys, _ := models.ZREVRANGEMoviesKey(key, start, stop)
 
 	for _, val := range movieKeys {
@@ -123,6 +136,9 @@ func MovieListsRange(key string, start, stop int64) []MovieListStruct {
 		mutex.Lock()
 		data = append(data, movieKeyMap)
 		mutex.Unlock()
+
+		byteData, _ := utils.Json.MarshalIndent(data, "", " ")
+		models.SaveMovies(cacheKey, string(byteData))
 	}
 
 	return data

utils/Spider.go

@@ -60,7 +60,6 @@ var (
 func StartSpider() {
 	// 获取所有分类
 	Categories := SpiderOKCategories()
-
 	for _, v := range Categories {
 
 		cateUrl := v.Link
@@ -219,13 +218,18 @@ func SpiderOKMovies(cateUrl string) {
 		})
 		defer p.Release()
 
-		for j := 0; j < lastPageInt; j++ {
+		for j := 1; j <= lastPageInt; j++ {
 
 			wg.Add(1)
 			pageUrl := CategoryToPageUrl(cateUrl, strconv.Itoa(j))
 
 			// todo 使用 goroutine 内存跟cpu消耗太高。 暂时没找到解决方案
 			ForeachPage(cateUrl, pageUrl)
+
+			// 完成一个分类删除所有页面缓存
+			if j == lastPageInt {
+				go DelAllListCacheKey()
+			}
 		}
 		wg.Wait()
 	})
@@ -556,3 +560,13 @@ func TransformId(Url string) string {
 
 	return strings.TrimRight(UrlStrSplit, ".html")
 }
+
+func DelAllListCacheKey() {
+
+	AllListCacheKey := RedisDB.Keys("movie_lists_key:detail_links:*").Val()
+
+	// 删除已经缓存的数据
+	for _, val := range AllListCacheKey {
+		RedisDB.Del(val)
+	}
+}