vendor/google.golang.org/appengine/log/log.go - gddo - Git at Google

 // Copyright 2011 Google Inc. All rights reserved.
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.

 /*
 Package log provides the means of writing and querying an application's logs
 from within an App Engine application.

 Example:
 	c := appengine.NewContext(r)
 	query := &log.Query{
 		AppLogs:  true,
 		Versions: []string{"1"},
 	}

 	for results := query.Run(c); ; {
 		record, err := results.Next()
 		if err == log.Done {
 			log.Infof(c, "Done processing results")
 			break
 		}
 		if err != nil {
 			log.Errorf(c, "Failed to retrieve next log: %v", err)
 			break
 		}
 		log.Infof(c, "Saw record %v", record)
 	}
 */
 package log // import "google.golang.org/appengine/log"

 import (
 	"errors"
 	"fmt"
 	"strings"
 	"time"

 	"github.com/golang/protobuf/proto"
 	"golang.org/x/net/context"

 	"google.golang.org/appengine"
 	"google.golang.org/appengine/internal"
 	pb "google.golang.org/appengine/internal/log"
 )

 // Query defines a logs query.
 type Query struct {
 	// Start time specifies the earliest log to return (inclusive).
 	StartTime time.Time

 	// End time specifies the latest log to return (exclusive).
 	EndTime time.Time

 	// Offset specifies a position within the log stream to resume reading from,
 	// and should come from a previously returned Record's field of the same name.
 	Offset []byte

 	// Incomplete controls whether active (incomplete) requests should be included.
 	Incomplete bool

 	// AppLogs indicates if application-level logs should be included.
 	AppLogs bool

 	// ApplyMinLevel indicates if MinLevel should be used to filter results.
 	ApplyMinLevel bool

 	// If ApplyMinLevel is true, only logs for requests with at least one
 	// application log of MinLevel or higher will be returned.
 	MinLevel int

 	// Versions is the major version IDs whose logs should be retrieved.
 	// Logs for specific modules can be retrieved by the specifying versions
 	// in the form "module:version"; the default module is used if no module
 	// is specified.
 	Versions []string

 	// A list of requests to search for instead of a time-based scan. Cannot be
 	// combined with filtering options such as StartTime, EndTime, Offset,
 	// Incomplete, ApplyMinLevel, or Versions.
 	RequestIDs []string
 }

 // AppLog represents a single application-level log.
 type AppLog struct {
 	Time    time.Time
 	Level   int
 	Message string
 }

 // Record contains all the information for a single web request.
 type Record struct {
 	AppID            string
 	ModuleID         string
 	VersionID        string
 	RequestID        []byte
 	IP               string
 	Nickname         string
 	AppEngineRelease string

 	// The time when this request started.
 	StartTime time.Time

 	// The time when this request finished.
 	EndTime time.Time

 	// Opaque cursor into the result stream.
 	Offset []byte

 	// The time required to process the request.
 	Latency     time.Duration
 	MCycles     int64
 	Method      string
 	Resource    string
 	HTTPVersion string
 	Status      int32

 	// The size of the request sent back to the client, in bytes.
 	ResponseSize int64
 	Referrer     string
 	UserAgent    string
 	URLMapEntry  string
 	Combined     string
 	Host         string

 	// The estimated cost of this request, in dollars.
 	Cost              float64
 	TaskQueueName     string
 	TaskName          string
 	WasLoadingRequest bool
 	PendingTime       time.Duration
 	Finished          bool
 	AppLogs           []AppLog

 	// Mostly-unique identifier for the instance that handled the request if available.
 	InstanceID string
 }

 // Result represents the result of a query.
 type Result struct {
 	logs        []*Record
 	context     context.Context
 	request     *pb.LogReadRequest
 	resultsSeen bool
 	err         error
 }

 // Next returns the next log record,
 func (qr *Result) Next() (*Record, error) {
 	if qr.err != nil {
 		return nil, qr.err
 	}
 	if len(qr.logs) > 0 {
 		lr := qr.logs[0]
 		qr.logs = qr.logs[1:]
 		return lr, nil
 	}

 	if qr.request.Offset == nil && qr.resultsSeen {
 		return nil, Done
 	}

 	if err := qr.run(); err != nil {
 		// Errors here may be retried, so don't store the error.
 		return nil, err
 	}

 	return qr.Next()
 }

 // Done is returned when a query iteration has completed.
 var Done = errors.New("log: query has no more results")

 // protoToAppLogs takes as input an array of pointers to LogLines, the internal
 // Protocol Buffer representation of a single application-level log,
 // and converts it to an array of AppLogs, the external representation
 // of an application-level log.
 func protoToAppLogs(logLines []*pb.LogLine) []AppLog {
 	appLogs := make([]AppLog, len(logLines))

 	for i, line := range logLines {
 		appLogs[i] = AppLog{
 			Time:    time.Unix(0, *line.Time*1e3),
 			Level:   int(*line.Level),
 			Message: *line.LogMessage,
 		}
 	}

 	return appLogs
 }

 // protoToRecord converts a RequestLog, the internal Protocol Buffer
 // representation of a single request-level log, to a Record, its
 // corresponding external representation.
 func protoToRecord(rl *pb.RequestLog) *Record {
 	offset, err := proto.Marshal(rl.Offset)
 	if err != nil {
 		offset = nil
 	}
 	return &Record{
 		AppID:             *rl.AppId,
 		ModuleID:          rl.GetModuleId(),
 		VersionID:         *rl.VersionId,
 		RequestID:         rl.RequestId,
 		Offset:            offset,
 		IP:                *rl.Ip,
 		Nickname:          rl.GetNickname(),
 		AppEngineRelease:  string(rl.GetAppEngineRelease()),
 		StartTime:         time.Unix(0, *rl.StartTime*1e3),
 		EndTime:           time.Unix(0, *rl.EndTime*1e3),
 		Latency:           time.Duration(*rl.Latency) * time.Microsecond,
 		MCycles:           *rl.Mcycles,
 		Method:            *rl.Method,
 		Resource:          *rl.Resource,
 		HTTPVersion:       *rl.HttpVersion,
 		Status:            *rl.Status,
 		ResponseSize:      *rl.ResponseSize,
 		Referrer:          rl.GetReferrer(),
 		UserAgent:         rl.GetUserAgent(),
 		URLMapEntry:       *rl.UrlMapEntry,
 		Combined:          *rl.Combined,
 		Host:              rl.GetHost(),
 		Cost:              rl.GetCost(),
 		TaskQueueName:     rl.GetTaskQueueName(),
 		TaskName:          rl.GetTaskName(),
 		WasLoadingRequest: rl.GetWasLoadingRequest(),
 		PendingTime:       time.Duration(rl.GetPendingTime()) * time.Microsecond,
 		Finished:          rl.GetFinished(),
 		AppLogs:           protoToAppLogs(rl.Line),
 		InstanceID:        string(rl.GetCloneKey()),
 	}
 }

 // Run starts a query for log records, which contain request and application
 // level log information.
 func (params *Query) Run(c context.Context) *Result {
 	req, err := makeRequest(params, internal.FullyQualifiedAppID(c), appengine.VersionID(c))
 	return &Result{
 		context: c,
 		request: req,
 		err:     err,
 	}
 }

 func makeRequest(params *Query, appID, versionID string) (*pb.LogReadRequest, error) {
 	req := &pb.LogReadRequest{}
 	req.AppId = &appID
 	if !params.StartTime.IsZero() {
 		req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
 	}
 	if !params.EndTime.IsZero() {
 		req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
 	}
 	if len(params.Offset) > 0 {
 		var offset pb.LogOffset
 		if err := proto.Unmarshal(params.Offset, &offset); err != nil {
 			return nil, fmt.Errorf("bad Offset: %v", err)
 		}
 		req.Offset = &offset
 	}
 	if params.Incomplete {
 		req.IncludeIncomplete = &params.Incomplete
 	}
 	if params.AppLogs {
 		req.IncludeAppLogs = &params.AppLogs
 	}
 	if params.ApplyMinLevel {
 		req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
 	}
 	if params.Versions == nil {
 		// If no versions were specified, default to the default module at
 		// the major version being used by this module.
 		if i := strings.Index(versionID, "."); i >= 0 {
 			versionID = versionID[:i]
 		}
 		req.VersionId = []string{versionID}
 	} else {
 		req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions))
 		for _, v := range params.Versions {
 			var m *string
 			if i := strings.Index(v, ":"); i >= 0 {
 				m, v = proto.String(v[:i]), v[i+1:]
 			}
 			req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{
 				ModuleId:  m,
 				VersionId: proto.String(v),
 			})
 		}
 	}
 	if params.RequestIDs != nil {
 		ids := make([][]byte, len(params.RequestIDs))
 		for i, v := range params.RequestIDs {
 			ids[i] = []byte(v)
 		}
 		req.RequestId = ids
 	}

 	return req, nil
 }

 // run takes the query Result produced by a call to Run and updates it with
 // more Records. The updated Result contains a new set of logs as well as an
 // offset to where more logs can be found. We also convert the items in the
 // response from their internal representations to external versions of the
 // same structs.
 func (r *Result) run() error {
 	res := &pb.LogReadResponse{}
 	if err := internal.Call(r.context, "logservice", "Read", r.request, res); err != nil {
 		return err
 	}

 	r.logs = make([]*Record, len(res.Log))
 	r.request.Offset = res.Offset
 	r.resultsSeen = true

 	for i, log := range res.Log {
 		r.logs[i] = protoToRecord(log)
 	}

 	return nil
 }

 func init() {
 	internal.RegisterErrorCodeMap("logservice", pb.LogServiceError_ErrorCode_name)
 }
	// Copyright 2011 Google Inc. All rights reserved.
	// Use of this source code is governed by the Apache 2.0
	// license that can be found in the LICENSE file.

	/*
	Package log provides the means of writing and querying an application's logs
	from within an App Engine application.

	Example:
	c := appengine.NewContext(r)
	query := &log.Query{
	AppLogs: true,
	Versions: []string{"1"},
	}

	for results := query.Run(c); ; {
	record, err := results.Next()
	if err == log.Done {
	log.Infof(c, "Done processing results")
	break
	}
	if err != nil {
	log.Errorf(c, "Failed to retrieve next log: %v", err)
	break
	}
	log.Infof(c, "Saw record %v", record)
	}
	*/
	package log // import "google.golang.org/appengine/log"

	import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"

	"google.golang.org/appengine"
	"google.golang.org/appengine/internal"
	pb "google.golang.org/appengine/internal/log"
	)

	// Query defines a logs query.
	type Query struct {
	// Start time specifies the earliest log to return (inclusive).
	StartTime time.Time

	// End time specifies the latest log to return (exclusive).
	EndTime time.Time

	// Offset specifies a position within the log stream to resume reading from,
	// and should come from a previously returned Record's field of the same name.
	Offset []byte

	// Incomplete controls whether active (incomplete) requests should be included.
	Incomplete bool

	// AppLogs indicates if application-level logs should be included.
	AppLogs bool

	// ApplyMinLevel indicates if MinLevel should be used to filter results.
	ApplyMinLevel bool

	// If ApplyMinLevel is true, only logs for requests with at least one
	// application log of MinLevel or higher will be returned.
	MinLevel int

	// Versions is the major version IDs whose logs should be retrieved.
	// Logs for specific modules can be retrieved by the specifying versions
	// in the form "module:version"; the default module is used if no module
	// is specified.
	Versions []string

	// A list of requests to search for instead of a time-based scan. Cannot be
	// combined with filtering options such as StartTime, EndTime, Offset,
	// Incomplete, ApplyMinLevel, or Versions.
	RequestIDs []string
	}

	// AppLog represents a single application-level log.
	type AppLog struct {
	Time time.Time
	Level int
	Message string
	}

	// Record contains all the information for a single web request.
	type Record struct {
	AppID string
	ModuleID string
	VersionID string
	RequestID []byte
	IP string
	Nickname string
	AppEngineRelease string

	// The time when this request started.
	StartTime time.Time

	// The time when this request finished.
	EndTime time.Time

	// Opaque cursor into the result stream.
	Offset []byte

	// The time required to process the request.
	Latency time.Duration
	MCycles int64
	Method string
	Resource string
	HTTPVersion string
	Status int32

	// The size of the request sent back to the client, in bytes.
	ResponseSize int64
	Referrer string
	UserAgent string
	URLMapEntry string
	Combined string
	Host string

	// The estimated cost of this request, in dollars.
	Cost float64
	TaskQueueName string
	TaskName string
	WasLoadingRequest bool
	PendingTime time.Duration
	Finished bool
	AppLogs []AppLog

	// Mostly-unique identifier for the instance that handled the request if available.
	InstanceID string
	}

	// Result represents the result of a query.
	type Result struct {
	logs []*Record
	context context.Context
	request *pb.LogReadRequest
	resultsSeen bool
	err error
	}

	// Next returns the next log record,
	func (qr Result) Next() (Record, error) {
	if qr.err != nil {
	return nil, qr.err
	}
	if len(qr.logs) > 0 {
	lr := qr.logs[0]
	qr.logs = qr.logs[1:]
	return lr, nil
	}

	if qr.request.Offset == nil && qr.resultsSeen {
	return nil, Done
	}

	if err := qr.run(); err != nil {
	// Errors here may be retried, so don't store the error.
	return nil, err
	}

	return qr.Next()
	}

	// Done is returned when a query iteration has completed.
	var Done = errors.New("log: query has no more results")

	// protoToAppLogs takes as input an array of pointers to LogLines, the internal
	// Protocol Buffer representation of a single application-level log,
	// and converts it to an array of AppLogs, the external representation
	// of an application-level log.
	func protoToAppLogs(logLines []*pb.LogLine) []AppLog {
	appLogs := make([]AppLog, len(logLines))

	for i, line := range logLines {
	appLogs[i] = AppLog{
	Time: time.Unix(0, line.Time1e3),
	Level: int(*line.Level),
	Message: *line.LogMessage,
	}
	}

	return appLogs
	}

	// protoToRecord converts a RequestLog, the internal Protocol Buffer
	// representation of a single request-level log, to a Record, its
	// corresponding external representation.
	func protoToRecord(rl pb.RequestLog) Record {
	offset, err := proto.Marshal(rl.Offset)
	if err != nil {
	offset = nil
	}
	return &Record{
	AppID: *rl.AppId,
	ModuleID: rl.GetModuleId(),
	VersionID: *rl.VersionId,
	RequestID: rl.RequestId,
	Offset: offset,
	IP: *rl.Ip,
	Nickname: rl.GetNickname(),
	AppEngineRelease: string(rl.GetAppEngineRelease()),
	StartTime: time.Unix(0, rl.StartTime1e3),
	EndTime: time.Unix(0, rl.EndTime1e3),
	Latency: time.Duration(rl.Latency) time.Microsecond,
	MCycles: *rl.Mcycles,
	Method: *rl.Method,
	Resource: *rl.Resource,
	HTTPVersion: *rl.HttpVersion,
	Status: *rl.Status,
	ResponseSize: *rl.ResponseSize,
	Referrer: rl.GetReferrer(),
	UserAgent: rl.GetUserAgent(),
	URLMapEntry: *rl.UrlMapEntry,
	Combined: *rl.Combined,
	Host: rl.GetHost(),
	Cost: rl.GetCost(),
	TaskQueueName: rl.GetTaskQueueName(),
	TaskName: rl.GetTaskName(),
	WasLoadingRequest: rl.GetWasLoadingRequest(),
	PendingTime: time.Duration(rl.GetPendingTime()) * time.Microsecond,
	Finished: rl.GetFinished(),
	AppLogs: protoToAppLogs(rl.Line),
	InstanceID: string(rl.GetCloneKey()),
	}
	}

	// Run starts a query for log records, which contain request and application
	// level log information.
	func (params Query) Run(c context.Context) Result {
	req, err := makeRequest(params, internal.FullyQualifiedAppID(c), appengine.VersionID(c))
	return &Result{
	context: c,
	request: req,
	err: err,
	}
	}

	func makeRequest(params Query, appID, versionID string) (pb.LogReadRequest, error) {
	req := &pb.LogReadRequest{}
	req.AppId = &appID
	if !params.StartTime.IsZero() {
	req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
	}
	if !params.EndTime.IsZero() {
	req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
	}
	if len(params.Offset) > 0 {
	var offset pb.LogOffset
	if err := proto.Unmarshal(params.Offset, &offset); err != nil {
	return nil, fmt.Errorf("bad Offset: %v", err)
	}
	req.Offset = &offset
	}
	if params.Incomplete {
	req.IncludeIncomplete = &params.Incomplete
	}
	if params.AppLogs {
	req.IncludeAppLogs = &params.AppLogs
	}
	if params.ApplyMinLevel {
	req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
	}
	if params.Versions == nil {
	// If no versions were specified, default to the default module at
	// the major version being used by this module.
	if i := strings.Index(versionID, "."); i >= 0 {
	versionID = versionID[:i]
	}
	req.VersionId = []string{versionID}
	} else {
	req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions))
	for _, v := range params.Versions {
	var m *string
	if i := strings.Index(v, ":"); i >= 0 {
	m, v = proto.String(v[:i]), v[i+1:]
	}
	req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{
	ModuleId: m,
	VersionId: proto.String(v),
	})
	}
	}
	if params.RequestIDs != nil {
	ids := make([][]byte, len(params.RequestIDs))
	for i, v := range params.RequestIDs {
	ids[i] = []byte(v)
	}
	req.RequestId = ids
	}

	return req, nil
	}

	// run takes the query Result produced by a call to Run and updates it with
	// more Records. The updated Result contains a new set of logs as well as an
	// offset to where more logs can be found. We also convert the items in the
	// response from their internal representations to external versions of the
	// same structs.
	func (r *Result) run() error {
	res := &pb.LogReadResponse{}
	if err := internal.Call(r.context, "logservice", "Read", r.request, res); err != nil {
	return err
	}

	r.logs = make([]*Record, len(res.Log))
	r.request.Offset = res.Offset
	r.resultsSeen = true

	for i, log := range res.Log {
	r.logs[i] = protoToRecord(log)
	}

	return nil
	}

	func init() {
	internal.RegisterErrorCodeMap("logservice", pb.LogServiceError_ErrorCode_name)
	}