question

Upvotes
Accepted
7 2 1 3

Why do I get uncompressed data ?

I am using the .NET SDK to make on demand extractions.

Because I set the AutomaticDecompression option to false, I would expect to receive compressed data (gzipped).

Instead :

1/ Without the "X-Direct-Download" header:
- TimeAndSalesExtractionRequest : I receive uncompressed data
- CorporateActionsStandardExtractionRequest : I receive uncompressed data

2/ With the "X-Direct-Download" header:
- TimeAndSalesExtractionRequest : I receive gzipped data (as expected)
- CorporateActionsStandardExtractionRequest : I receive uncompressed data

Here is my code :

    public static class Cases
    {
        public static void Run()
        {
            Uri dssUri = new Uri("https://hosted.datascopeapi.reuters.com/RestApi/1/");
            string dssUserName = "myUsername";
            string dssUserPassword = "myPassword";


            ExtractionsContext extractionsContext = new ExtractionsContext(dssUri, dssUserName, dssUserPassword);
            //extractionsContext.DefaultRequestHeaders.Add("X-Direct-Download", "true");
            extractionsContext.Options.AutomaticDecompression = false;


            //-----------------------------------------------------------------
            //Define Extraction Request
            //-----------------------------------------------------------------
            ExtractionRequestBase extractionRequest = GetTimeAndSalesExtractionRequest();
            //ExtractionRequestBase extractionRequest = GetCorporateActionsStandardExtractionRequest();


            //-----------------------------------------------------------------
            //Extraction
            //NOTE: if the extraction request takes more than 30 seconds the async mechanism will be used,
            //the SDK handles this automatically for you.
            //-----------------------------------------------------------------
            RawExtractionResult extractionResult = extractionsContext.ExtractRaw(extractionRequest);


            //-----------------------------------------------------------------
            //Write data to file:
            //-----------------------------------------------------------------
            DssStreamResponse response = extractionsContext.GetReadStream(extractionResult);
            using (var fileStream = File.Create("data.csv.gz"))
            {
                response.Stream.CopyTo(fileStream);
            }
        }


        public static ExtractionRequestBase GetTimeAndSalesExtractionRequest()
        {
            var instru1 = InstrumentIdentifier.Create(IdentifierType.Ric, "SOGN.PA");
            var instru2 = InstrumentIdentifier.Create(IdentifierType.Ric, "BNPP.PA");
            var instru3 = InstrumentIdentifier.Create(IdentifierType.Ric, "CAGR.PA");
            var instru4 = InstrumentIdentifier.Create(IdentifierType.Ric, "HSBA.L");
            InstrumentIdentifier[] instrumentIdentifierArray = new InstrumentIdentifier[] { instru1, instru2, instru3, instru4 };


            string[] requestedFieldNames = {
                "Trade - Price", "Trade - Volume", "Trade - Qualifiers",
                "Quote - Bid Price", "Quote - Bid Size", "Quote - Number of Buyers",
                "Quote - Ask Price", "Quote - Ask Size", "Quote - Number of Sellers",
                "Auction - Price", "Auction - Volume"
            };


            TickHistoryTimeAndSalesCondition condition = new TickHistoryTimeAndSalesCondition
            {
                ReportDateRangeType = ReportDateRangeType.Range,
                QueryStartDate = new DateTimeOffset(2017, 01, 10, 6, 0, 0, TimeSpan.FromHours(0)),
                QueryEndDate = new DateTimeOffset(2017, 01, 10, 10, 0, 0, TimeSpan.FromHours(0)),
                ApplyCorrectionsAndCancellations = false,
                ExtractBy = TickHistoryExtractByMode.Ric,
                MessageTimeStampIn = TickHistoryTimeOptions.LocalExchangeTime,
                SortBy = TickHistorySort.SingleByTimestamp,
                DisplaySourceRIC = false
            };


            TickHistoryTimeAndSalesExtractionRequest extractionRequest = new TickHistoryTimeAndSalesExtractionRequest
            {
                IdentifierList = InstrumentIdentifierList.Create(instrumentIdentifierArray),
                ContentFieldNames = requestedFieldNames,
                Condition = condition
            };


            return extractionRequest;
        }


        public static ExtractionRequestBase GetCorporateActionsStandardExtractionRequest()
        {
            var instru1 = InstrumentIdentifier.Create(IdentifierType.Ric, "SOGN.PA");
            var instru2 = InstrumentIdentifier.Create(IdentifierType.Ric, "BNPP.PA");
            var instru3 = InstrumentIdentifier.Create(IdentifierType.Ric, "CAGR.PA");
            var instru4 = InstrumentIdentifier.Create(IdentifierType.Ric, "HSBA.L");
            InstrumentIdentifier[] instrumentIdentifierArray = new InstrumentIdentifier[] { instru1, instru2, instru3, instru4 };


            string[] requestedFieldNames = {
                "RIC", "Dividend Ex Date", "Dividend Rate", "Dividend Currency"
            };


            CorporateActionsStandardCondition condition = new CorporateActionsStandardCondition
            {
                PreviousDays = 90,
                ReportDateRangeType = ReportDateRangeType.Delta,
                CorporateActionsCapitalChangeType = CorporateActionsCapitalChangeType.CapitalChangeExDate,
                CorporateActionsDividendsType = CorporateActionsDividendsType.DividendPayDate,
                CorporateActionsEarningsType = CorporateActionsEarningsType.PeriodEndDate,
                CorporateActionsEquityOfferingsType = CorporateActionsEquityOfferingsType.AllPendingDeals,
                CorporateActionsMergersAcquisitionsType = CorporateActionsMergersAcquisitionsType.DealAnnouncementDate,
                CorporateActionsNominalValueType = CorporateActionsNominalValueType.NominalValueDate,
                CorporateActionsSharesType = CorporateActionsSharesType.SharesAmountDate,
                CorporateActionsStandardEventsType = CorporateActionsStandardEventsType.None,
                CorporateActionsVotingRightsType = CorporateActionsVotingRightsType.VotingRightsDate,
                QueryStartDate = null,
                NextDays = null,
                QueryEndDate = null,
                PendingEventsHours = null,
                PendingEventsMinutes = null,
                IncludeInstrumentsWithNoEvents = null,
                IncludeNullDates = null,
                ExcludeDeletedEvents = true,
                IncludeCapitalChangeEvents = true,
                IncludeDividendEvents = true,
                IncludeEarningsEvents = true,
                IncludeMergersAndAcquisitionsEvents = true,
                IncludeNominalValueEvents = true,
                IncludePublicEquityOfferingsEvents = true,
                IncludeSharesOutstandingEvents = true,
                IncludeVotingRightsEvents = true
            };


            CorporateActionsStandardExtractionRequest extractionRequest = new CorporateActionsStandardExtractionRequest
            {
                IdentifierList = InstrumentIdentifierList.Create(instrumentIdentifierArray),
                ContentFieldNames = requestedFieldNames,
                Condition = condition
            };


            return extractionRequest;
        }
    }
tick-history-rest-api
icon clock
10 |1500

Up to 2 attachments (including images) can be used with a maximum of 5.0 MiB each and 10.0 MiB total.

1 Answer

Upvote
Accepted
38.1k 71 35 53

When not using X-Direct-Download: False, the response from DSS usually contains Content-Encoding: gzip which causes the proxy server decompresses the data and sends CSV to the application.

On the other hand, when using X-Direct-Download: True, the response from AWS doesn't have this header so the proxy server will not decompress the data and send raw gzip to the application.

For CorporateActionsStandardExtractionRequest with X-Direct-Download: True, currently this feature is available for the following custom Tick History reports: Tick History Time and Sales, Tick History Market Depth, Tick History Intraday Summaries, and Tick History Raw reports.

Therefore, when using CorporateActionsStandardExtractionRequest, it will always get the data from DSS server which contains Content-Encoding: gzip in the response.

This Content-Encoding: gzip in the response from DSS has been reported to the DSS team to verify the problem. For more information, please refer to this question.


gzip.png (40.0 KiB)
aws.png (30.0 KiB)
icon clock
10 |1500

Up to 2 attachments (including images) can be used with a maximum of 5.0 MiB each and 10.0 MiB total.

Click below to post an Idea Post Idea