patternsqlMinor
Lead/Lag Implementation in SQL Server 2008 R2: Max memory exceeded
Viewed 0 times
2008sqllagmaxmemoryleadserverimplementationexceeded
Problem
Background
I'm trying to establish a "visits" sequence, wherein if an animal is detected at essentially the same place (
where e.g. all the A(n) locations belong to
As
I'm running into memory issues (not to mention computation time) with the following:
```
WITH s AS (
SELECT
RANK() OVER (PARTITION BY det.Technology, det.XmitID ORDER BY DetectDate ASC, ReceiverID ASC) as DetID,
COALESCE(TA.AnimalID, det.Technology+'-'+cast(da.XmitID AS nvarchar), 'BSVALUE999') as AnimalID,
det.Technology, det.XmitID, DetectDate, det.location as Location, RL.General_Location as GLocation, ReceiverID
FROM
Detections_with_Location as det JOIN
Receiver_Locations as RL
ON det.Location=RL.Location LEFT OUTER JOIN
Tagged_Animal as TA
ON det.AnimalID=TA.AnimalID
)
INSERT INTO ##ttOrder_det (AnimalID, Technology, XmitID, DD, Location, GLocation, ReceiverID, DetID, PrevDD, BinInc)
SELECT
s1.AnimalID, --was a coalesce
s1.Technology, s1.XmitID, s1.DetectDate, s1.Location, s1.GLocation, s1.ReceiverID,
s1.DetID,
sLAG.DetectDate,
CASE WHEN sLAG.DetectDate IS NULL
THEN 1
ELSE CASE WHEN sLAG.GLocation = s1.GLocation
THEN 0
ELSE 1
END
END AS BinInc
FROM s as s1
LEFT OUTER JOIN s AS sLAG ON
s1.DetID = sLAG.DetID + 1
I'm trying to establish a "visits" sequence, wherein if an animal is detected at essentially the same place (
General_Location) it counts as a single visit, but if it goes elsewhere and then back, that's an additional visit to the same location. So, if animal is detected in a Location sequenceA1, A2, A3, A3, A3, A1, B2, D4, A2where e.g. all the A(n) locations belong to
General_Location "A", this would have the first 6 detections as visit 1 (@A), next as visit 2 (@B), next as visit 3 (@D), next as visit 4 (back @A). As
LAG and LEAD are not available in SQL Server 2008R2 (nor is UNBOUNDED PRECEDING in the PARTITIONing clause), I'm attempting a work-around as described in this SQL Authority blog entry.I'm running into memory issues (not to mention computation time) with the following:
```
WITH s AS (
SELECT
RANK() OVER (PARTITION BY det.Technology, det.XmitID ORDER BY DetectDate ASC, ReceiverID ASC) as DetID,
COALESCE(TA.AnimalID, det.Technology+'-'+cast(da.XmitID AS nvarchar), 'BSVALUE999') as AnimalID,
det.Technology, det.XmitID, DetectDate, det.location as Location, RL.General_Location as GLocation, ReceiverID
FROM
Detections_with_Location as det JOIN
Receiver_Locations as RL
ON det.Location=RL.Location LEFT OUTER JOIN
Tagged_Animal as TA
ON det.AnimalID=TA.AnimalID
)
INSERT INTO ##ttOrder_det (AnimalID, Technology, XmitID, DD, Location, GLocation, ReceiverID, DetID, PrevDD, BinInc)
SELECT
s1.AnimalID, --was a coalesce
s1.Technology, s1.XmitID, s1.DetectDate, s1.Location, s1.GLocation, s1.ReceiverID,
s1.DetID,
sLAG.DetectDate,
CASE WHEN sLAG.DetectDate IS NULL
THEN 1
ELSE CASE WHEN sLAG.GLocation = s1.GLocation
THEN 0
ELSE 1
END
END AS BinInc
FROM s as s1
LEFT OUTER JOIN s AS sLAG ON
s1.DetID = sLAG.DetID + 1
Solution
I'm submitting this as an answer, as I am currently avoiding the out-of-memory type errors as well as have reduced the time of run significantly (was 4+ hours, ending in failure; now is 1.25 hrs, ending in success). However, I'm sure that after about 10 million more records, this problem may reoccur, so I would appreciate any additional comments or answers aimed at making this more memory-efficient for the future.
The "solution" to this point was to drop unneeded fields and especially indices from the design of the temp tables. Additionally, index creation for non-constraint keys was deferred until after the table was filled.
To address the issue (pointed out first by @MartinSmith) of partitioning not matching the later
Code for the Q
Quick comments
I switched to using
Again, as two users pointed out in the comments above, I'm not using the full PK of the
I was planning on dropping the
at the same location (one receiver was presumed lost but later found) which did indeed detect the same animal at the same time
Follow-up code to complete the task
```
CREATE TABLE ##ttOrder_det2 (
AnimalID nvarchar(50) not null,
DetID int NOT NULL,
BinNum int NULL,
CONSTRAINT [PK_ttRDA2] PRIMARY KEY CLUSTERED
([AnimalID] ASC, [DetID] ASC)
The "solution" to this point was to drop unneeded fields and especially indices from the design of the temp tables. Additionally, index creation for non-constraint keys was deferred until after the table was filled.
To address the issue (pointed out first by @MartinSmith) of partitioning not matching the later
JOIN, I created a field in a subquery which I then used for both PARTITIONing and for JOINing.Code for the Q
set nocount on;
SET XACT_ABORT ON;
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'started' as reason; --last run 2015-12-16 18:22:02
GO
CREATE TABLE ##ttOrder_det (
AnimalID nvarchar(50) not null,
DD DateTime not null,
ReceiverID int NOT NULL,
Location nvarchar(255) NOT NULL,
GLocation nvarchar(255) NULL,
DetID int NOT NULL,
BinIncORNum int NULL, -- first pass=Inc, second pass=Num
CONSTRAINT [PK_ttRDA] PRIMARY KEY CLUSTERED
([AnimalID] ASC, [DD] ASC, ReceiverID ASC)
WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY];
GO
CREATE UNIQUE INDEX UX_CTR ON ##ttOrder_det (AnimalID ASC, DetID ASC);
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'created first temporary table' as reason; --last run 2015-12-16 18:22:02
GO
WITH s AS (
SELECT
AnimalID, DetectDate,
ReceiverID, Location, GLocation,
ROW_NUMBER() OVER (
PARTITION BY AnimalID ORDER BY DetectDate ASC, ReceiverID ASC
) as DetID
FROM (
SELECT
COALESCE (
TF.AnimalID,
Det.Technology+'-'+cast(Det.XmitID AS nvarchar(10)),
Det.AnimalID,
N'BSVALUE999'
) as AnimalID,
DetectDate,
ReceiverID,
COALESCE (
Det.location,
N'Unknown Location'
) as Location,
COALESCE (
ML.General_Location,
N'Invalid General Location - Orphaned Receiver'
) as GLocation
FROM
Detections_with_Location as Det LEFT OUTER JOIN
Receiver_Locations as ML ON Det.Location=ML.Location LEFT OUTER JOIN
Tagged_Animal as TF ON Det.AnimalID=TF.AnimalID
) AS T
)
INSERT INTO ##ttOrder_det (AnimalID, DD, ReceiverID, Location, GLocation, DetID, BinIncORNum)
SELECT
s1.AnimalID,
s1.DetectDate, s1.ReceiverID, s1.Location, s1.GLocation,
s1.DetID,
CASE WHEN sLg.DetectDate IS NULL
THEN 1
ELSE CASE WHEN sLg.GLocation = s1.GLocation
THEN 0
ELSE 1
END
END AS BinInc
FROM s as s1
LEFT OUTER JOIN s AS sLg ON
s1.AnimalID= sLg.AnimalID AND
s1.DetID = sLg.DetID + 1
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'filled first temp table' as reason,
COUNT(*) as SizeOfFirstTemp FROM ##ttOrder_det; --2015-12-16 18:43:03, 46627879
GO
CREATE NONCLUSTERED INDEX NIX_F on ##ttOrder_det (AnimalID ASC);
CREATE NONCLUSTERED INDEX NIX_R ON ##ttOrder_det (DetID ASC);
--dropped several additional indices: `NIX_`s VTC, TCD, A, DD, Bi
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'indexed first temp table' as reason; --2015-12-16 18:44:12
GOQuick comments
I switched to using
ROW_NUMBER instead of RANK. This is non-deterministic, but at least will result in no "ties", which would result in breaking of the LAG implementation around that point. Ties shouldn't exist, but this is just future-proofing against General_Locations that are very close together that co-detect the same transmission.Again, as two users pointed out in the comments above, I'm not using the full PK of the
Tagged_Animal table, so there's a possibility that there be a JOIN with an ambiguous AnimalID. However, at present, both AnimalID and TagSN are UNIQUE, albeit unconstrained.I was planning on dropping the
ReceiverID field in favor of using the Location, but I had a period in which there were two Receivers deployedat the same location (one receiver was presumed lost but later found) which did indeed detect the same animal at the same time
Follow-up code to complete the task
```
CREATE TABLE ##ttOrder_det2 (
AnimalID nvarchar(50) not null,
DetID int NOT NULL,
BinNum int NULL,
CONSTRAINT [PK_ttRDA2] PRIMARY KEY CLUSTERED
([AnimalID] ASC, [DetID] ASC)
Code Snippets
set nocount on;
SET XACT_ABORT ON;
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'started' as reason; --last run 2015-12-16 18:22:02
GO
CREATE TABLE ##ttOrder_det (
AnimalID nvarchar(50) not null,
DD DateTime not null,
ReceiverID int NOT NULL,
Location nvarchar(255) NOT NULL,
GLocation nvarchar(255) NULL,
DetID int NOT NULL,
BinIncORNum int NULL, -- first pass=Inc, second pass=Num
CONSTRAINT [PK_ttRDA] PRIMARY KEY CLUSTERED
([AnimalID] ASC, [DD] ASC, ReceiverID ASC)
WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY];
GO
CREATE UNIQUE INDEX UX_CTR ON ##ttOrder_det (AnimalID ASC, DetID ASC);
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'created first temporary table' as reason; --last run 2015-12-16 18:22:02
GO
WITH s AS (
SELECT
AnimalID, DetectDate,
ReceiverID, Location, GLocation,
ROW_NUMBER() OVER (
PARTITION BY AnimalID ORDER BY DetectDate ASC, ReceiverID ASC
) as DetID
FROM (
SELECT
COALESCE (
TF.AnimalID,
Det.Technology+'-'+cast(Det.XmitID AS nvarchar(10)),
Det.AnimalID,
N'BSVALUE999'
) as AnimalID,
DetectDate,
ReceiverID,
COALESCE (
Det.location,
N'Unknown Location'
) as Location,
COALESCE (
ML.General_Location,
N'Invalid General Location - Orphaned Receiver'
) as GLocation
FROM
Detections_with_Location as Det LEFT OUTER JOIN
Receiver_Locations as ML ON Det.Location=ML.Location LEFT OUTER JOIN
Tagged_Animal as TF ON Det.AnimalID=TF.AnimalID
) AS T
)
INSERT INTO ##ttOrder_det (AnimalID, DD, ReceiverID, Location, GLocation, DetID, BinIncORNum)
SELECT
s1.AnimalID,
s1.DetectDate, s1.ReceiverID, s1.Location, s1.GLocation,
s1.DetID,
CASE WHEN sLg.DetectDate IS NULL
THEN 1
ELSE CASE WHEN sLg.GLocation = s1.GLocation
THEN 0
ELSE 1
END
END AS BinInc
FROM s as s1
LEFT OUTER JOIN s AS sLg ON
s1.AnimalID= sLg.AnimalID AND
s1.DetID = sLg.DetID + 1
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'filled first temp table' as reason,
COUNT(*) as SizeOfFirstTemp FROM ##ttOrder_det; --2015-12-16 18:43:03, 46627879
GO
CREATE NONCLUSTERED INDEX NIX_F on ##ttOrder_det (AnimalID ASC);
CREATE NONCLUSTERED INDEX NIX_R ON ##ttOrder_det (DetID ASC);
--dropped several additional indices: `NIX_`s VTC, TCD, A, DD, Bi
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'indexed first temp table' as reason; --2015-12-16 18:44:12
GOCREATE TABLE ##ttOrder_det2 (
AnimalID nvarchar(50) not null,
DetID int NOT NULL,
BinNum int NULL,
CONSTRAINT [PK_ttRDA2] PRIMARY KEY CLUSTERED
([AnimalID] ASC, [DetID] ASC)
WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY];
GO
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'second temp table created' as reason; --2015-12-16 18:44:15
GO
-- SET XACT_ABORT ON will cause the transaction to be uncommittable
-- when the constraint violation occurs.
BEGIN TRANSACTION
BEGIN TRY
DECLARE @AnimalID as nvarchar(50);
DECLARE @DetID as int;
DECLARE @BinInc as int;
DECLARE @BinNum as int;
DECLARE @AnimalVisit as CURSOR;
SET @AnimalVisit = CURSOR FOR SELECT AnimalID, DetID, BinIncORNum FROM ##ttOrder_det ORDER BY AnimalID, DetID;
OPEN @AnimalVisit;
FETCH NEXT FROM @AnimalVisit INTO @AnimalID, @DetID, @BinInc;
WHILE @@FETCH_STATUS = 0
BEGIN
IF (@DetID <= 1) SET @BinNum = 0;
SET @BinNum += @BinInc;
INSERT INTO ##ttOrder_det2 (AnimalID, DetID, BinNum) VALUES (@AnimalID, @DetID, @BinNum);
FETCH NEXT FROM @AnimalVisit INTO @AnimalID, @DetID, @BinInc;
END
CLOSE @AnimalVisit;
DEALLOCATE @AnimalVisit;
END TRY
BEGIN CATCH
exec sp_lock; -- added to display the open locks after the timeout
-- exec sp_who2; -- shows the active processes
EXECUTE usp_GetErrorInfo;
--RETURN -- ignoring this error for brevity
-- Test XACT_STATE:
-- If 1, the transaction is committable.
-- If -1, the transaction is uncommittable and should
-- be rolled back.
-- XACT_STATE = 0 means that there is no transaction and
-- a commit or rollback operation would generate an error.
-- Test whether the transaction is uncommittable.
IF (XACT_STATE()) = -1
BEGIN
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'The transaction is in an uncommittable state. Rolling back transaction.' as reason
SET XACT_ABORT off
RETURN -- 1 --error
ROLLBACK TRANSACTION;
END;
-- Test whether the transaction is committable.
IF (XACT_STATE()) = 1
BEGIN
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'The transaction is committable. Committing transaction.' as reason
COMMIT TRANSACTION;
END;
END CATCH;
IF @@TRANCOUNT > 0
COMMIT TRANSACTION;
GO
SELECT
cast(cast(current_timestamp as datetime2) as varchar(19)) as time,
'filled second temp table' as reason,
COUNT(*) as Table2Size from ##ttOrder_det2; --2015-12-16 19:11:17, 46627879
GO
CREATE NONCLUSTERED INDEX NIX_CT2 ON ##ttOrder_det2 (AnimalID ASC);
CREATE NONCLUSTERED INDEX NIX_R2 ON ##ttOrder_det2 (DetID ASC);
SELECT
cast(cast(current_timestamp as dContext
StackExchange Database Administrators Q#123481, answer score: 5
Revisions (0)
No revisions yet.