1  package journalistd
  2  
  3  import (
  4  	"context"
  5  	"strconv"
  6  	"time"
  7  
  8  	"github.com/google/uuid"
  9  	"go.uber.org/zap"
 10  
 11  	"github.com/mrusme/journalist/ent"
 12  	"github.com/mrusme/journalist/ent/feed"
 13  	"github.com/mrusme/journalist/ent/item"
 14  	"github.com/mrusme/journalist/ent/user"
 15  
 16  	"github.com/mrusme/journalist/lib"
 17  	"github.com/mrusme/journalist/rss"
 18  )
 19  
 20  var VERSION string
 21  
 22  type Journalistd struct {
 23  	jctx *lib.JournalistContext
 24  
 25  	config    *lib.Config
 26  	entClient *ent.Client
 27  	logger    *zap.Logger
 28  
 29  	daemonStop          chan bool
 30  	autoRefreshInterval time.Duration
 31  }
 32  
 33  func Version() string {
 34  	return VERSION
 35  }
 36  
 37  func New(
 38  	jctx *lib.JournalistContext,
 39  ) (*Journalistd, error) {
 40  	jd := new(Journalistd)
 41  	jd.jctx = jctx
 42  	jd.config = jctx.Config
 43  	jd.entClient = jctx.EntClient
 44  	jd.logger = jctx.Logger
 45  
 46  	if err := jd.initAdminUser(); err != nil {
 47  		return nil, err
 48  	}
 49  
 50  	interval, err := strconv.Atoi(jd.config.Feeds.AutoRefresh)
 51  	if err != nil {
 52  		jd.logger.Error(
 53  			"Feeds.AutoRefresh is not a valid number (seconds)",
 54  			zap.Error(err),
 55  		)
 56  		return nil, err
 57  	}
 58  	jd.autoRefreshInterval = time.Duration(interval)
 59  
 60  	return jd, nil
 61  }
 62  
 63  func (jd *Journalistd) IsDebug() bool {
 64  	debug, err := strconv.ParseBool(jd.config.Debug)
 65  	if err != nil {
 66  		return false
 67  	}
 68  
 69  	return debug
 70  }
 71  
 72  func (jd *Journalistd) initAdminUser() error {
 73  	var admin *ent.User
 74  	var err error
 75  
 76  	admin, err = jd.entClient.User.
 77  		Query().
 78  		Where(user.Username(jd.config.Admin.Username)).
 79  		Only(context.Background())
 80  	if err != nil {
 81  		admin, err = jd.entClient.User.
 82  			Create().
 83  			SetUsername(jd.config.Admin.Username).
 84  			SetPassword(jd.config.Admin.Password).
 85  			SetRole("admin").
 86  			Save(context.Background())
 87  		if err != nil {
 88  			jd.logger.Error(
 89  				"Failed query/create admin user",
 90  				zap.Error(err),
 91  			)
 92  			return err
 93  		}
 94  	}
 95  
 96  	if admin.Password == "admin" {
 97  		jd.logger.Debug(
 98  			"Admin user",
 99  			zap.String("username", admin.Username),
100  			zap.String("password", admin.Password),
101  		)
102  	} else {
103  		jd.logger.Debug(
104  			"Admin user",
105  			zap.String("username", admin.Username),
106  			zap.String("password", "xxxxxx"),
107  		)
108  	}
109  
110  	return nil
111  }
112  
113  func (jd *Journalistd) Start() bool {
114  	jd.logger.Info(
115  		"Starting Journalist daemon",
116  	)
117  	jd.daemonStop = make(chan bool)
118  	go jd.daemon()
119  	return true
120  }
121  
122  func (jd *Journalistd) Stop() {
123  	jd.logger.Info(
124  		"Stopping Journalist daemon",
125  	)
126  	jd.daemonStop <- true
127  }
128  
129  func (jd *Journalistd) daemon() {
130  	jd.logger.Debug(
131  		"Journalist daemon started, looping",
132  	)
133  	for {
134  		select {
135  		case <-jd.daemonStop:
136  			jd.logger.Debug(
137  				"Journalist daemon loop ended",
138  			)
139  			return
140  		default:
141  			jd.logger.Debug(
142  				"RefreshAll starting, refreshing all feeds",
143  			)
144  			errs := jd.RefreshAll()
145  			if len(errs) > 0 {
146  				jd.logger.Error(
147  					"RefreshAll completed with errors",
148  					zap.Errors("errors", errs),
149  				)
150  			} else {
151  				jd.logger.Debug(
152  					"RefreshAll completed",
153  				)
154  			}
155  			time.Sleep(time.Second * jd.autoRefreshInterval)
156  		}
157  	}
158  }
159  
160  func (jd *Journalistd) RefreshAll() []error {
161  	var errs []error
162  
163  	dbFeeds, err := jd.entClient.Feed.
164  		Query().
165  		All(context.Background())
166  	if err != nil {
167  		errs = append(errs, err)
168  		return errs
169  	}
170  
171  	var feedIds []uuid.UUID = make([]uuid.UUID, len(dbFeeds))
172  	for i, dbFeed := range dbFeeds {
173  		feedIds[i] = dbFeed.ID
174  	}
175  
176  	return jd.Refresh(feedIds)
177  }
178  
179  func (jd *Journalistd) Refresh(feedIds []uuid.UUID) []error {
180  	var errs []error
181  
182  	dbFeeds, err := jd.entClient.Feed.
183  		Query().
184  		Where(
185  			feed.IDIn(feedIds...),
186  		).
187  		WithItems(func(query *ent.ItemQuery) {
188  			query.
189  				Select(item.FieldItemGUID).
190  				Where(item.CrawlerContentHTMLNEQ(""))
191  		}).
192  		All(context.Background())
193  	if err != nil {
194  		errs = append(errs, err)
195  		return errs
196  	}
197  
198  	for _, dbFeed := range dbFeeds {
199  		var exceptItemGUIDs []string
200  		for _, exceptItem := range dbFeed.Edges.Items {
201  			exceptItemGUIDs = append(exceptItemGUIDs, exceptItem.ItemGUID)
202  		}
203  
204  		rc, errr := rss.NewClient(
205  			dbFeed.URL,
206  			dbFeed.Username,
207  			dbFeed.Password,
208  			true,
209  			exceptItemGUIDs,
210  			jd.logger,
211  		)
212  		if len(errr) > 0 {
213  			errs = append(errs, errr...)
214  			continue
215  		}
216  
217  		dbFeedTmp := jd.entClient.Feed.
218  			Create()
219  		rc.SetFeed(
220  			dbFeed.URL,
221  			dbFeed.Username,
222  			dbFeed.Password,
223  			dbFeedTmp,
224  		)
225  		feedID, err := dbFeedTmp.
226  			OnConflictColumns("url", "username", "password").
227  			UpdateNewValues().
228  			ID(context.Background())
229  		if err != nil {
230  			errs = append(errs, err)
231  		}
232  
233  		dbItems := make([]*ent.ItemCreate, len(rc.Feed.Items))
234  		for i := 0; i < len(rc.Feed.Items); i++ {
235  			dbItems[i] = jd.entClient.Item.
236  				Create()
237  			dbItems[i] = rc.SetItem(
238  				feedID,
239  				i,
240  				dbItems[i],
241  			)
242  		}
243  		err = jd.entClient.Item.
244  			CreateBulk(dbItems...).
245  			OnConflictColumns("item_guid").
246  			UpdateNewValues().
247  			Exec(context.Background())
248  		if err != nil {
249  			errs = append(errs, err)
250  		}
251  	}
252  
253  	return errs
254  }