question

Upvotes
Accepted
s61670 avatar image
89 ●2 ●6 ●9

downloads only 8 minutes instead of 1 hour

RIC: ESH1m (Raw Legacy Market Depth) time 

QueryStartDate = new DateTimeOffset(2021, 03, 05, 03, 0, 0, TimeSpan.Zero),//startDate,
                                QueryEndDate = new DateTimeOffset(2021, 03, 05, 04, 0, 0, TimeSpan.Zero),//lastDate,

Only downloads for 8 minutes !!! Why?

code:

using System;
using System.Linq;
using System.Diagnostics;
using System.IO;
using ThomsonReuters.Dss.Api.StandardExtractions;
using ThomsonReuters.Dss.Api;
using ThomsonReuters.Dss.Api.Content;
using ThomsonReuters.Dss.Api.Extractions;
using ThomsonReuters.Dss.Api.Extractions.ExtractionRequests;
using ThomsonReuters.Dss.Api.Extractions.ReportExtractions;
using ThomsonReuters.Dss.Api.Extractions.ReportTemplates;
using ThomsonReuters.Dss.Api.Extractions.Schedules;
using ThomsonReuters.Dss.Api.Extractions.SubjectLists;
using ThomsonReuters.Dss.Api.Search;
using ThomsonReuters.Dss.Core.RestApi;

namespace RefinitivHistoryTesting
{
    class Program
    {
        static void Main(string[] args)
        {
            //-----------------------------------------------------------------
            //Connect and authenticate to the DSS server:
            //-----------------------------------------------------------------
            Uri dssUri = new Uri("https://hosted.datascopeapi.reuters.com/RestApi/v1/");
            string dssUserName = "*****";
            string dssUserPassword = "*****";

            var out_dir = @".\Storage\";
            var SearchContext = new SearchContext(dssUri, dssUserName, dssUserPassword);
            var ExtractionsContext = new ExtractionsContext(dssUri, dssUserName, dssUserPassword);
            var availableMbPFields = ExtractionsContext.GetValidContentFieldTypes(ReportTemplateTypes.TickHistoryRaw);
            var availableLL2Fields = ExtractionsContext.GetValidContentFieldTypes(ReportTemplateTypes.TickHistoryMarketDepth);
            var availableTnSFields = ExtractionsContext.GetValidContentFieldTypes(ReportTemplateTypes.TickHistoryTimeAndSales);


            ExtractionsContext.Options.AutomaticDecompression = true;

            var dateRange = DateTimeRange.Create(
                new DateTimeOffset(2008, 01, 01, 0, 0, 0, TimeSpan.Zero),
                new DateTimeOffset(2020, 12, 22, 0, 0, 0, TimeSpan.Zero));

            var request = new HistoricalChainResolutionRequest
            {
                ChainRics = new[] { "0#ES:" },//{ "0#1URO:" },
                Range = dateRange
            };

            var chain_results = SearchContext.HistoricalChainResolution(request);
            foreach (var chain in chain_results)
            {
                Console.WriteLine("Chain Identifier: {0} {1}, Constituents: {2}", chain.IdentifierType, chain.Identifier, chain.Constituents.Count());
                foreach (var constituent in chain.Constituents)
                {
                    var searchResults = SearchContext.HistoricalSearch(
                        HistoricalSearchRequest.Create(
                            constituent.Identifier, constituent.IdentifierType, dateRange, HistoricalResultsBy.Ric, null));

                    //foreach (var result in from pos in searchResults where pos.DomainCode == "8" select pos)
                    //foreach (var result in searchResults)

                    var result = searchResults.FirstOrDefault(pos => pos.Identifier.Contains("ESH1"));
                    if (result != null)
                    {
                        var startDate = result.FirstDate;//new DateTimeOffset(2021, 03, 09, 0, 0, 0, TimeSpan.Zero);//result.FirstDate;
                        var lastDate = result.LastDate;//new  DateTimeOffset(2021, 03, 09, 23, 59, 59, TimeSpan.Zero);//result.LastDate;
                        Console.WriteLine("{0},\t {1},\t {2},\t {3},\t {4}", result.Identifier, result.IdentifierType, startDate, lastDate, result.DomainCode);
    
               var extractionResult = ExtractionsContext.ExtractRaw(
                        new TickHistoryMarketDepthExtractionRequest
                        {
                            Condition = new TickHistoryMarketDepthCondition
                            {
                                ReportDateRangeType = ReportDateRangeType.Range,
                                QueryStartDate = new DateTimeOffset(2021, 03, 05, 03, 0, 0, TimeSpan.Zero),//startDate,
                                QueryEndDate = new DateTimeOffset(2021, 03, 05, 04, 0, 0, TimeSpan.Zero),//lastDate,
                                ExtractBy = TickHistoryExtractByMode.Ric,
                                MessageTimeStampIn = TickHistoryTimeOptions.GmtUtc,
                                SortBy = TickHistorySort.SingleByRic,
                                View = TickHistoryMarketDepthViewOptions.LegacyLevel2,
                                DisplaySourceRIC = true

                            },
                             //ContentFieldNames = availableLL2Fields.Select(f => f.Name).ToArray(),
                             IdentifierList = new InstrumentIdentifierList
                            {
                                InstrumentIdentifiers = new[]
                                {
                                     InstrumentIdentifier.Create( IdentifierType.Ric,"ESH1m")
                                },
                                ValidationOptions = new InstrumentValidationOptions
                                {
                                    AllowHistoricalInstruments = true
                                }
                            },
                       
                    });

                        using (var response = ExtractionsContext.RawExtractionResultOperations.GetReadStream(extractionResult))
                        using (var reader = new StreamReader(response.Stream))
                        {
                            var fileName = out_dir + "_" + result.Identifier + "_" + startDate.DateTime.ToString("dd.MM.yyyy") + "_" + lastDate.DateTime.ToString("dd.MM.yyyy") + "_MarketByPrice.txt";
                            using (var fileStream = File.Create(fileName))
                                response.Stream.CopyTo(fileStream);
                        }
                        
                    }
                }
            }

            Console.WriteLine("Press Enter to exit");
            Console.ReadLine();
        }
    }

}


tick-history-rest-api
icon clock
10 |1500

Up to 2 attachments (including images) can be used with a maximum of 5.0 MiB each and 10.0 MiB total.

2 Answers

Upvote
Accepted
Jirapongse avatar image
38.1k ●71 ●35 ●53

@s61670

There is an issue when downloading a large file when the automatic decompression is enabled.

You need to disable it, download it, and then decompress it. The sample code is available in the .Net SDK Tutorial 5: On Demand extraction, file IO.

First, disable the AutomaticDecompression.

var ExtractionsContext = new ExtractionsContext(dssUri, dssUserName, dssUserPassword);             
ExtractionsContext.Options.AutomaticDecompression = false;

Then, use the below code to save a file.

using ICSharpCode.SharpZipLib.GZip;
...
string line;
var sw = new StreamWriter("Storage\\output.csv", false);
var streamResponse = ExtractionsContext.GetReadStream(extractionResult);
{
    using (GZipInputStream gzip = new GZipInputStream(streamResponse.Stream))
    {
        using (StreamReader reader = new StreamReader(gzip, Encoding.UTF8))
        {
            while ((line = reader.ReadLine()) != null)
            {
                sw.WriteLine(line);                            
            }                     
        }
    }
}
sw.Close();


icon clock
10 |1500

Up to 2 attachments (including images) can be used with a maximum of 5.0 MiB each and 10.0 MiB total.

Upvotes
s61670 avatar image
89 ●2 ●6 ●9

I try to download for an hour, downloads randomly.

One hour downloads a file where 282460 lines in another hour downloads a file where 3077397 lines



45667.png (55.5 KiB)
icon clock
10 |1500

Up to 2 attachments (including images) can be used with a maximum of 5.0 MiB each and 10.0 MiB total.

Click below to post an Idea Post Idea