# Datastream User Created Timeseries
# ----------------------------------
#
# Datastream permits users to create and manage custom items. One of these
# types is custom timeseries data. Clients can upload their own timeseries
# data to Datastream’s central systems. These can then be used in combination
# with Datastream maintained series in charts and data requests with the full
# power of the Datastream expressions language.
#
# DatastreamUserCreated_TimeSeries.R defines a TimeseriesClient class and
# ancillary classes that assist in creating and managing timeseries data.
#
# This simple test script demonstrates how to manage custom timeseries data
# using the DatastreamUserCreated_TimeSeries.R module.
#
# Prerequesites:
#
# Firstly, your Datastream credentials need to be permissioned to use the
# Datastream API services. To verify you have access, please access the API
# test page at https://product.datastream.com/dswsclient/Docs/TestUserCreatedItems.aspx.
# Selecting the "Get Token" service method, insert your Datastream credentials
# and press the Execute button. If your credentials are valid and are permissioned
# to access the API service, you will receive a token response. If you are not
# permissioned, you will receive below message
# "User 'Your ID' not entitled to ClientApi service.".
# Please contact your LSEG representative to get your ID permissioned.
#
# Once you have permissions to access the API service, you also need to be
# permissioned to specifically access the service that permits you to access
# and manage custom user created items such as timeseries. You can verify this
# by using the "Get All Items" service method on the above test page. Using the
# token returned from the GetToken method, copy the token into the Token field
# and then select Execute. If you are permissioned for the user created items
# service, you will get a response listing the timeseries available to you on
# Datastream. This may be an empty list if you are a new customer or have never created user created items before. If you
# are not permissioned, you will receive an error message:
# "ID 'Your ID' is not permissioned for user created items.".
# Please contact your LSEG representative to get your ID permissioned.
#

library (DatastreamR)
library(logger)

# You can set the threshold for log messages as given below
# logger::log_threshold(ERROR)

# This is a simple support function for displaying the contents of an individual
# timeseries item. It's used frequently in the demos below.
ProcessTimeseriesResponse = function(tsResponse, tsName)
{
  if (!is.null(tsResponse))
  {
    # Any request dealing with a single user created item returns a DSUserObjectResponse. This has ResponseStatus property that indicates success or failure
    if (tsResponse$ResponseStatus != DSUserObjectResponseStatus$UserObjectSuccess)
      print(paste('Request failed for timeseries', tsName, 'with error ', names(DSUserObjectResponseStatus)[tsResponse$ResponseStatus + 1], ': ', tsResponse$ErrorMessage))
    else if (!is.null(tsResponse$UserObject))  # The timeseries item won't be returned if you set SkipItem true in CreateItem or UpdateItem
    {
      # Here we simply display the timeseries data using a dataframe.
      tsItem = tsResponse$UserObject
      metadata = c (Id = tsItem$Id,
                    Desc = tsItem$Description,
                    LastModified = as.character(tsItem$LastModified),
                    StartDate = ifelse (!is.null(tsItem$DateInfo), as.character(tsItem$DateInfo$StartDate), NULL),
                    EndDate = ifelse(!is.null(tsItem$DateInfo),as.character(tsItem$DateInfo$EndDate), NULL),
                    Frequency = ifelse(!is.null(tsItem$DateInfo), names(DSUserObjectFrequency)[tsItem$DateInfo$Frequency + 1], NULL),
                    NoOfValues = ifelse(!is.null(tsItem$DateRange), tsItem$DateRange$ValuesCount , 0))
      df = data.frame(metadata)
      print(df)

      if (!is.null(tsItem$DateRange))
      {
        df = data.frame(Dates = sapply(tsItem$DateRange$Dates, FUN = function(x) { return (as.character(x)) }),
                        Values = sapply(tsItem$DateRange$Values, FUN = function(x) { ifelse (is.null(x), return (NA_character_ ), return (x) )} ))
        # Values if NULL, is printed as <NA> because, while converting list to vector
        # either by using as.vector or sapply, the NULL values in the list are deleted.
        # and thus there will be mismatch in no of rows and cannot be put in a dataframe
        print(df)
      }
    }
  }
  else
    print(paste('Timeseries ', tsName, ' successfully updated but item details not returned.'))
}


#
# To query and modify timeseries, you first need to create a TimeseriesClient
# object using your Datastream credentials.
#
# This can be done in two ways. The first method is simply to create the
# client passing your Datastream credentials in directly:
#
#     timeseriesClient = TimeseriesClient(NULL, 'YourID', 'YourPwd')
#
#     Alternatively, you can use a configuration file containing your
#     credentials. The config file just needs your credentials in the following
#     format:
#
#     [credentials]
#     username=YourID
#     password=YourPwd
#
#     Then create the TimeseriesClient object passing in the config file.
#
#     timeseriesClient = TimeseriesClient('config.ini')
#
#
# Okay, let's start the demo by creating the TimeseriesClient.

timeseriesClient = NULL

# Assuming you have permissioned credentials, replace 'YourID' and 'YourPwd'
# with them and then let's logon and create the TimeseriesClient object.
# Note the basic error handling for DSUserObjectFault exceptions

tryCatch({
  # First step is creating the client by replacing 'YourID' and
  # 'YourPwd' with your own credentials.
  print('Creating a TimeseriesClient object using your credentials.')

  timeseriesClient = TimeSeriesClient$new(NULL, "zdsm043", "horse485")

  #timeseriesClient = TimeSeriesClient('Config.ini')

},
error = function(e)
{
  stop(message(e))
}
)

if (!is.null(timeseriesClient) & timeseriesClient$IsValid())
{
  print(paste('Successfully created the TimeseriesClient.'))
  # With a valid client connection, we can start to perfom some actions
  # dealing with timeseries items.

  readline("Press Enter to continue...")


  # You still need exception handling for possible network errors
  tryCatch({
    # First step is to get a list of all the timeseries you
    # currently have available. This list may be empty if you
    # are a new user.
    print('Requesting all timeseries available for your datastream
                       ID.')
    itemsResp = timeseriesClient$GetAllItems()

    # DSUserObjectGetAllResponse has ResponseStatus property that
    # indicates success or failure for the query

    if (!is.null(itemsResp))
    {
      if (itemsResp$ResponseStatus != DSUserObjectResponseStatus$UserObjectSuccess)
      {
        # Your Datastream Id might not be permissioned for
        # managing user created items on this API
        print(paste('GetAllItems failed with error',
                    names(DSUserObjectResponseStatus)[itemsResp$ResponseStatus+1],
                    itemsResp$ErrorMessage))
      }

      else if ((is.null(itemsResp$UserObjects)) | (itemsResp$UserObjectsCount == 0))
        print(paste('GetAllItems returned zero timeseries items.'))
      else
      {
        # You do have access to some timeseries
        # Note, because you can have thousands of timeseries
        # per Datastream ID, and each of these can contain upto
        # 130 years of daily data, the GetAllItems method returns
        # just the summary data for each timeseries. Each of the
        # DSTimeSeriesResponseObjects returned in the UserObjects
        # collection will specify the number of datapoints available in
        # the ValuesCount field of the DateRange property, however the Dates
        # and Values fields will always be NULL or an empty list. To view
        # the actual dates and values of a specific timeseries, you need to
        # use the GetItem method. Here we just put the timeseries details into
        # a dataframe and list them

        print(paste('GetAllItems returned',
                    itemsResp$UserObjectsCount, 'timeseries items.'))
        colnames = c('Id', 'Desc', 'LastModified', 'startDate', 'endDate','Frequency',
                     'NoOfValues')
        df = data.frame()
        for (tsItem in itemsResp$UserObjects)
        {
          coldata = NULL
          if (!is.null(tsItem))
          {
            rowdata = c (Id = tsItem$Id,
                         Desc = tsItem$Description,
                         LastModified = as.character(tsItem$LastModified),
                         StartDate = ifelse (!is.null(tsItem$DateInfo), as.character(tsItem$DateInfo$StartDate), NULL),
                         EndDate = ifelse(!is.null(tsItem$DateInfo),as.character(tsItem$DateInfo$EndDate), NULL),
                         Frequency = ifelse(!is.null(tsItem$DateInfo), names(DSUserObjectFrequency)[tsItem$DateInfo$Frequency + 1], NULL),
                         NoOfValues = ifelse(!is.null(tsItem$DateRange), tsItem$DateRange$ValuesCount , 0))
            df = rbind(df, rowdata)
          }
        }
        colnames(df) = colnames
        print(df)
      }


      readline("Press Enter to continue...")


      # Let us request and display the full details for the first timeseries
      # in your returned list (if there is one)

      if (!is.null(itemsResp) &
          itemsResp$ResponseStatus == DSUserObjectResponseStatus$UserObjectSuccess &
          itemsResp$UserObjectsCount > 0 &
          !is.null(itemsResp$UserObjects))
      {
        firstItem = itemsResp$UserObjects[[1]]$Id
        print(paste('Requesting the first timeseries in your list: ',
                    firstItem ))
        getItemResp = timeseriesClient$GetItem(firstItem)

        # Let's use ProcessTimeseriesResponse to perfom general processing
        # of the DSUserObjectResponse as we will be handling a few timeseries
        # responses. See the comments in ProcessTimeseriesResponse for a
        # description of handling a response
        ProcessTimeseriesResponse(getItemResp, firstItem)

      }
    }


    readline("Press Enter to continue...")


    # Let's now start the process of creating a new timeseries.
    # First of all, we need to know which dates are supported on Datastream.
    # Datastream only stores the actual working weekdays for a given date
    # range, sovwe need to provide values that match the dates supported.

    # The API supports a GetTimeseriesDateRange method which returns a list
    # of supported dates for a given start date, end date and frequency.

    # We'll start off creating a timeseries with quarterly frequency, so let's
    # retrieve the supported dates.
    # This example intentionally shows two different date constructors,
    # requesting quarterly dates between 2016 and 2022.

    startDate = as.Date.character('2016-1-1', format = "%Y-%m-%d")
    endDate = as.Date('2022-04-01')
    freq = DSUserObjectFrequency$Quarterly
    print('Requesting GetTimeseriesDateRange to get supported dates from 2016-01-01 to 2022-04-01 at quarterly frequency.')
    dateRangeResp = timeseriesClient$GetTimeseriesDateRange(startDate, endDate, freq)

    #process the returned dates
    datesCount = 0 # we'll use this later in our example for creating an actual timeseries
    if (is.null(dateRangeResp))
    {
      if (dateRangeResp$ResponseStatus != DSUserObjectResponseStatus$UserObjectSuccess)
        print('GetTimeseriesDateRange failed with error ', names(DSUserObjectResponseStatus)[dateRangeResp$ResponseStatus + 1],': ', dateRangeResp$ErrorMessage)
    }
    else if (!is.null(dateRangeResp$Dates))
    {
      df = data.frame(Dates = sapply(dateRangeResp$Dates,
                                     FUN = function(x) { return (as.character(x)) }))
      print(df)
    }

    # You would normally use the returned dates to populate the timeseries
    # request's objects Values field with corresponding values.
    # Here we are just going to track how many valid dates there were within
    #the requested period.
    datesCount = length(dateRangeResp$Dates)

    readline("Press Enter to continue...")


    # With the list of supported dates returned from GetTimeseriesDateRange,
    # you would then create an array of values associated
    # with the given dates from your data source.
    # For this example, we'll just populate an array of test values randomly
    # between 10.00 and 200.00 with two decimal places
    datesCount = ifelse (datesCount > 0, datesCount, 26)    # safety: setting datesCount in case GetTimeseriesDateRange response processing was skipped over

    # We'll simply create an array of values datesCount long with random
    # values between 10 and 200 with 2 decimal places to reflect data
    # retrieved from some source
    values = runif(datesCount, 10, 200)

    # We'll use a variable to store the test ID as we'll manipulate the
    # timeseries a few times.
    # Note Timeseries IDs must be 8 uppercase alphanumeric characters in
    # length and start with TS. e.g. TSZZZ001
    # Feel free to change this ID to something you prefer.
    testID = 'TSVVV023'

    # To now create the timeseries, we need a DSTimeSeriesRequestObject.
    # The constructor for this requires you to provide only the mandatory
    # properties. There are some key parameters you need to set in a timeseries
    # request object: The ID, the start and end dates, the frequency of data
    # and the data values for the given date range and frequency

    # You can construct the timeseries with the key data in two ways:
    # First method is to use the optional parameters in the constructor to
    # supply the key data
    #testTs = DSTimeSeriesRequestObject$new(testID, startDate, endDate,
                                       #DSUserObjectFrequency$Quarterly, values)

    # Or you can construct the basic object and populate the fields directly
    testTs = DSTimeSeriesRequestObject$new()
    testTs$Id = testID
    testTs$DataInput = DSTimeSeriesDataInput$new() # or more directly testTs$DataInput = DSTimeSeriesDataInput$new(startDate, endDate, DSUserObjectFrequency$Quarterly, values)
    testTs$DataInput$StartDate = startDate
    testTs$DataInput$EndDate = endDate
    testTs$DataInput$Frequency = DSUserObjectFrequency$Quarterly
    testTs$DataInput$Values = values


    #Set any other optional properties directly
    testTs$DisplayName = 'My first test timeseries' # if you don't set the DisplayName it will be set to the same as the ID in the response
    testTs$DecimalPlaces = 2 # we created our array of values with 2 decimal places. You can specify 0 to 8 decimal places
    testTs$Units = "Billions"  # Leave units blank or set with any custom text (max 12 chars)
    testTs$DateAlignment = DSTimeSeriesDateAlignment$MidPeriod # when requested by users in data retrieval, you can specify the quarterly dates to be returned as start, middle or end of the select period (frequency)

    # Now we actually try and create the item.
    # When you create an item, the create request will normally be rejected if the item already exists. However, there is an option to overwrite the existing
    # item. We will use that option here in case the item was created in an earlier test and not subsequently deleted.
    print(paste('Creating a new timeseries', testID))
    tsResp = timeseriesClient$CreateItem(testTs, overWrite = TRUE)

    # The timeseries should now be returned in the list of timeseries if you
    # call timeseriesClient.GetAllItems() again.
    # It should also, after a few minutes, appear in Datastream's Navigator tool.
    # You should also be able to chart it using the DSChart tool in Eikon or
    # LSEG Workstation. The raw data should also be retrievable with
    # requests via the standard Datastream API or via Datastream.
    # For Office (DSGrid queries).
    # Here, we will just display the response returned from the mainframe.
    ProcessTimeseriesResponse(tsResp, testID)

    # You can pause here and test the new item in DFO, Navigator or DSCharting.
    readline("Press Enter to continue...")

    print("Re-creating the same Timeseries")
    values = runif(30, 20, 200)
    testTs = DSTimeSeriesRequestObject$new(testID, as.Date("2024-01-01"), as.Date("2024-02-01"),
                                           DSUserObjectFrequency$Daily, values)
    tsResp = timeseriesClient$CreateItem(testTs, overWrite = TRUE)
    ProcessTimeseriesResponse(tsResp, testID)
    readline("Press Enter to continue...")


    # Now we can try updating an existing item.
    # Let's modify it to be a daily frequency series with data from 2000
    # until 2020.
    # Again, we'll get the list of supported dates between the start and
    # end date.
    startDate = as.Date('2000-01-01')
    endDate = as.Date('2020-12-31')
    freq = DSUserObjectFrequency$Daily
    print('Requesting GetTimeseriesDateRange to get supported dates from 2001-01-01 to 2020-12-31 at daily frequency.')
    dateRangeResp = timeseriesClient$GetTimeseriesDateRange(startDate, endDate, freq)
    print('')

    # With the list of supported dates returned from GetTimeseriesDateRange,
    # you would then create an array of values associated with the given dates
    # from your data source
    # Again, for this example, we'll just populate an array of test values
    # randomly between 10.00 and 200.00 with two decimal places
    if (!(is.null(dateRangeResp)) &
        dateRangeResp$ResponseStatus == DSUserObjectResponseStatus$UserObjectSuccess &
        !(is.null(dateRangeResp$Dates)))
    {
      # We'll just take the count of the number of dates and generate random
      # data to simulate us retrieving corresponding values from your data source.
      datesCount = length(dateRangeResp$Dates)
      values = runif(datesCount, 10, 200) # an array datesCount long with random values between 10 and 200 with 2 decimal places
    }

    # set the new dates, frequency and values on our existing timeseries object

    # Note: Datastream takes the StartDate, Frequency and Values and creates
    # the timeseries based only using these parameters. The EndDate is not
    # actually used internally other than for logging purposes. The true end
    # date is calculated based on the start date, frequency and list of values. Supply too few or too many values
    # and the mainframe will accept them and set the end date accordingly
    # based on the given frequency for the item.
    # To demonstrate this, we deliberately set the enddate to be a date other
    # than 2020-12-31. The response item will give the true end date as
    # calculated (2020-12-31)
    testTs$DataInput = DSTimeSeriesDataInput$new(startDate, as.Date('2005-12-31'),
                                             freq, values)
    testTs$DisplayName = 'Modifying timeseries to be daily frequency'
    testTs$DataInput$Frequency = DSUserObjectFrequency$Daily

    # and finally call UpdateItem
    print(paste('Updating our timeseries', testID, 'to have daily frequency.'))
    tsResp = timeseriesClient$UpdateItem(testTs)
    # Again, we will just display the updated timeseries object returned
    # from the mainframe.
    ProcessTimeseriesResponse(tsResp, testID)

    # You can pause here and test the updated item in DFO, Navigator or
    # DSCharting.
    readline("Press Enter to continue...")


    # Now we can create and modify timeseries, let's look at how we handle
    # non-trading days. On Datastream non-trading days are stored as float NaN
    # values. R supports NaN values and JSON (used to communicate with
    # API server) also support NaN values.
    # The client can provide NULL input values for Non-trading days, similar to
    # python code, which internally are converted to NaN Values.

    # However, the JSON response received from API server will be NULL values
    # for non-trading days. By default, the client expects non-trading values
    # to be represented by NULL value. An option has been provided for the
    # client to choose values to be returned for non-trading days to be NaN.
    # We'll now demonstrate both modes of operation.


    # First of all, let's demo the default mode where trading days are accepted
    # and returned only as NULL values.
    # Again, we'll modify the same test item. We'll keep all previous settings
    # the same but simply modify the values to include some NULL values
    testTs$DataInput$Values = list (30.01, 50.02, NULL, 70.04, 80.05, NULL, 100.07)

    # You can provide NaN in place of NULL, as shown below
    # testTs$DataInput$Values = list (30.01, 50.02, NaN, 70.04, 80.05, NaN, 100.07)

    # One modification we have to do is set the CarryIndicator to 'No' to stop
    # the mainframe replacing the NaNs with the prior values (padding)
    testTs$CarryIndicator = DSTimeSeriesCarryIndicator$No

    # And apply the update
    print(paste('Updating our timeseries', testID,
                'to have some non-trading values represented by NULL.'))
    tsResp = timeseriesClient$UpdateItem(testTs, skipItemReturn = TRUE)
    # Again, we will just display the updated timeseries object returned from
    # the mainframe.
    # The returned Values for non-trading days should be set to NULL.
    ProcessTimeseriesResponse(tsResp, testID)

    # You can pause here and test the updated item in DFO, Navigator or
    # DSCharting.
    readline("Press Enter to continue...")



    # We can now demonstrate the alternative option of using float NaN to
    # retrieve non-trading values as NaNs.
    # To do this we set useNaNforNotANumber to be True
    timeseriesClient$useNaNforNotANumber = TRUE

    # In the following we just demonstrate providing floating NaN values and
    # NULL when creating or updating a timeseries.
    testTs$DataInput$Values = list (50.01, 60.02, NaN, 80.04, NaN, 90.05, NULL, 100.07, NULL, 120.23)

    # And apply the update
    print(paste('Updating our timeseries', testID,
                'to have some non-trading values represented by NaNs.'))
    tsResp = timeseriesClient$UpdateItem(testTs)
    # Again, we will just display the updated timeseries object returned from
    # the mainframe.
    # The returned Values for non-trading days should be set to NaN values.
    ProcessTimeseriesResponse(tsResp, testID)

    # You can pause here and test the updated item in DFO, Navigator or
    # DSCharting.
    readline("Press Enter to continue...")


    # We are now done testing. Let's clean up the example by deleting it
    print(paste('Deleting our timeseries', testID))
    delResp = timeseriesClient$DeleteItem(testID)
    if (!is.null(delResp))
    {
      if (delResp$ResponseStatus != DSUserObjectResponseStatus$UserObjectSuccess)
        print(paste('Timeseries DeleteItem failed on', delResp.UserObjectId, ' with error', names(DSUserObjectResponseStatus)[delResp$ResponseStatus + 1], ':', delResp$ErrorMessage))
      else
        print(paste('Timeseries', delResp$UserObjectId,'successfully deleted.'))
    }

    # You can pause here and test the deleted item in DFO, Navigator or
    # DSCharting.
    readline("Press Enter to continue...")


    # let's try that again and see an example of the error handling as
    # DeleteItem will fail
    print(paste('Deleting our timeseries', testID, 'again to demonstrate the API server returning an error response.'))
    delResp = timeseriesClient$DeleteItem(testID)
    if (!is.null(delResp))
    {
      if (delResp$ResponseStatus != DSUserObjectResponseStatus$UserObjectSuccess)
        print(paste('Timeseries DeleteItem failed on', delResp$UserObjectId,
                    'with error',
                    names(DSUserObjectResponseStatus)[delResp$ResponseStatus + 1],
                    ':', delResp$ErrorMessage))
      else # delResp.UserObject won't be set on DeleteItem if successful
        print(paste('Timeseries', delResp$UserObjectId,'successfully deleted.'))
    }
  },
  error = function(e)
  {
    print(paste('TimeseriesClient request failed with error message:', message(e)))
    stop(message(e))
  })
}
# The demo has now finished.

print(exp)

print('The demo has now finished.')