Return to Answer

create table #X
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  Version int not null,
  RowData xml not null
);

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);


insert into #X(PK, UpdateDate, Version, RowData)
select bt.PK,
       bt.UpdateDate,
       0,
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable as bt
union all
select bt.PK,
       bt.UpdateDate,
       row_number() over(partition by bt.PK order by bt.UpdateDate desc),
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable_archive as bt;

with C as 
(
  select X.PK,
         X.UpdateDate,
         X.Version,
         T.C.value('local-name(.)', 'nvarchar(128)') as ColumnName,
         T.C.value('text()[1]', 'varchar(max)') as Value
  from #X as X
    cross apply X.RowData.nodes('*') as T(C)
)
insert into #T (PK, UpdateDate, ColumnName, Value, Version)
select C.PK,
       C.UpdateDate,
       C.ColumnName,
       C.Value,
       C.Version
from C 
where C.ColumnName not in (N'PK', N'UpdateDate');

/*
option (querytraceon 8649);

The above query might need some trick to go parallel.
For the testdata I had on my machine exection time is 16 seconds vs 2 seconds
https://web.archive.org/web/20180404164406/http://sqlblog.com/blogs/paul_white/archive/2011/12/23/forcing-a-parallel-query-execution-plan.aspx
http://dataeducation.com/next-level-parallel-plan-forcing-an-alternative-to-8649/

*/

select New.PK,
       New.UpdateDate,
       New.ColumnName,
       Old.Value as OldValue,
       New.Value as NewValue
from #T as New
  left outer join #T as Old
    on Old.PK = New.PK and
       Old.ColumnName = New.ColumnName and
       Old.Version = New.Version + 1;

create table #X
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  Version int not null,
  RowData xml not null
);

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);


insert into #X(PK, UpdateDate, Version, RowData)
select bt.PK,
       bt.UpdateDate,
       0,
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable as bt
union all
select bt.PK,
       bt.UpdateDate,
       row_number() over(partition by bt.PK order by bt.UpdateDate desc),
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable_archive as bt;

with C as 
(
  select X.PK,
         X.UpdateDate,
         X.Version,
         T.C.value('local-name(.)', 'nvarchar(128)') as ColumnName,
         T.C.value('text()[1]', 'varchar(max)') as Value
  from #X as X
    cross apply X.RowData.nodes('*') as T(C)
)
insert into #T (PK, UpdateDate, ColumnName, Value, Version)
select C.PK,
       C.UpdateDate,
       C.ColumnName,
       C.Value,
       C.Version
from C 
where C.ColumnName not in (N'PK', N'UpdateDate');

/*
option (querytraceon 8649);

The above query might need some trick to go parallel.
For the testdata I had on my machine exection time is 16 seconds vs 2 seconds
https://sqlkiwi.blogspot.com/2011/12/forcing-a-parallel-query-execution-plan.html
http://dataeducation.com/next-level-parallel-plan-forcing-an-alternative-to-8649/

*/

select New.PK,
       New.UpdateDate,
       New.ColumnName,
       Old.Value as OldValue,
       New.Value as NewValue
from #T as New
  left outer join #T as Old
    on Old.PK = New.PK and
       Old.ColumnName = New.ColumnName and
       Old.Version = New.Version + 1;

create table #X
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  Version int not null,
  RowData xml not null
);

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);


insert into #X(PK, UpdateDate, Version, RowData)
select bt.PK,
       bt.UpdateDate,
       0,
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable as bt
union all
select bt.PK,
       bt.UpdateDate,
       row_number() over(partition by bt.PK order by bt.UpdateDate desc),
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable_archive as bt;

with C as 
(
  select X.PK,
         X.UpdateDate,
         X.Version,
         T.C.value('local-name(.)', 'nvarchar(128)') as ColumnName,
         T.C.value('text()[1]', 'varchar(max)') as Value
  from #X as X
    cross apply X.RowData.nodes('*') as T(C)
)
insert into #T (PK, UpdateDate, ColumnName, Value, Version)
select C.PK,
       C.UpdateDate,
       C.ColumnName,
       C.Value,
       C.Version
from C 
where C.ColumnName not in (N'PK', N'UpdateDate');

/*
option (querytraceon 8649);

The above query might need some trick to go parallel.
For the testdata I had on my machine exection time is 16 seconds vs 2 seconds
http://sqlblog.com/blogs/paul_white/archive/2011/12/23/forcing-a-parallel-query-execution-plan.aspx
http://sqlblog.com/blogs/adam_machanic/archive/2013/07/11/next-level-parallel-plan-porcing.aspx

*/

select New.PK,
       New.UpdateDate,
       New.ColumnName,
       Old.Value as OldValue,
       New.Value as NewValue
from #T as New
  left outer join #T as Old
    on Old.PK = New.PK and
       Old.ColumnName = New.ColumnName and
       Old.Version = New.Version + 1;

create table #X
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  Version int not null,
  RowData xml not null
);

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);


insert into #X(PK, UpdateDate, Version, RowData)
select bt.PK,
       bt.UpdateDate,
       0,
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable as bt
union all
select bt.PK,
       bt.UpdateDate,
       row_number() over(partition by bt.PK order by bt.UpdateDate desc),
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable_archive as bt;

with C as 
(
  select X.PK,
         X.UpdateDate,
         X.Version,
         T.C.value('local-name(.)', 'nvarchar(128)') as ColumnName,
         T.C.value('text()[1]', 'varchar(max)') as Value
  from #X as X
    cross apply X.RowData.nodes('*') as T(C)
)
insert into #T (PK, UpdateDate, ColumnName, Value, Version)
select C.PK,
       C.UpdateDate,
       C.ColumnName,
       C.Value,
       C.Version
from C 
where C.ColumnName not in (N'PK', N'UpdateDate');

/*
option (querytraceon 8649);

The above query might need some trick to go parallel.
For the testdata I had on my machine exection time is 16 seconds vs 2 seconds
https://web.archive.org/web/20180404164406/http://sqlblog.com/blogs/paul_white/archive/2011/12/23/forcing-a-parallel-query-execution-plan.aspx
http://dataeducation.com/next-level-parallel-plan-forcing-an-alternative-to-8649/

*/

select New.PK,
       New.UpdateDate,
       New.ColumnName,
       Old.Value as OldValue,
       New.Value as NewValue
from #T as New
  left outer join #T as Old
    on Old.PK = New.PK and
       Old.ColumnName = New.ColumnName and
       Old.Version = New.Version + 1;

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);
 
create unique clustered index IX_T on #T(PK, ColumnName, Version);

You could match the rows to find new and old value with a self join on PK, ColumnName and Version = Version + 1.

That should give you what I believe is a decent query plan with a merge join.

Or if your server memory can handle it you can leave out the clustered index and get a Hash Match parallel plan instead.

The not so pretty part is, of course, doing the unpivot of your 300 columns into the temp table from the two base tables.

create table #X
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  RowData xml not null
);

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);


insert into #X(PK, UpdateDate, RowData)
select bt.PK,
       bt.UpdateDate,
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable as bt
union all
select bt.PK,
       bt.UpdateDate,
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable_archive as bt;

with C as 
(
  select X.PK,
         X.UpdateDate,
         T.C.value('local-name(.)', 'nvarchar(128)') as ColumnName,
         T.C.value('text()[1]', 'varchar(max)') as Value
  from #X as X
    cross apply X.RowData.nodes('*') as T(C)
)
insert into #T (PK, UpdateDate, ColumnName, Value, Version)
select C.PK,
       C.UpdateDate,
       C.ColumnName,
       C.Value,
       row_number() over(partition by C.PK, C.ColumnName order by C.UpdateDate desc)
from C;

--create unique clustered index IX_T on #T(PK, ColumnName, Version);

select New.PK,
       New.UpdateDate,
       New.ColumnName,
       Old.Value as OldValue,
       New.Value as NewValue
from #T as New
  left outer join #T as Old
    on Old.PK = New.PK and
       Old.ColumnName = New.ColumnName and
       Old.Version = New.Version + 1;

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);

You could match the rows to find new and old value with a self join on PK, ColumnName and Version = Version + 1.

The not so pretty part is, of course, doing the unpivot of your 300 columns into the temp table from the two base tables.

create table #X
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  Version int not null,
  RowData xml not null
);

create table #T
(
  PK varchar(12) not null,
  UpdateDate datetime not null,
  ColumnName nvarchar(128) not null,
  Value varchar(max),
  Version int not null
);


insert into #X(PK, UpdateDate, Version, RowData)
select bt.PK,
       bt.UpdateDate,
       0,
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable as bt
union all
select bt.PK,
       bt.UpdateDate,
       row_number() over(partition by bt.PK order by bt.UpdateDate desc),
       (select bt.* for xml path(''), elements xsinil, type)
from dbo.bigtable_archive as bt;

with C as 
(
  select X.PK,
         X.UpdateDate,
         X.Version,
         T.C.value('local-name(.)', 'nvarchar(128)') as ColumnName,
         T.C.value('text()[1]', 'varchar(max)') as Value
  from #X as X
    cross apply X.RowData.nodes('*') as T(C)
)
insert into #T (PK, UpdateDate, ColumnName, Value, Version)
select C.PK,
       C.UpdateDate,
       C.ColumnName,
       C.Value,
       C.Version
from C 
where C.ColumnName not in (N'PK', N'UpdateDate');

/*
option (querytraceon 8649);

The above query might need some trick to go parallel.
For the testdata I had on my machine exection time is 16 seconds vs 2 seconds
http://sqlblog.com/blogs/paul_white/archive/2011/12/23/forcing-a-parallel-query-execution-plan.aspx
http://sqlblog.com/blogs/adam_machanic/archive/2013/07/11/next-level-parallel-plan-porcing.aspx

*/

select New.PK,
       New.UpdateDate,
       New.ColumnName,
       Old.Value as OldValue,
       New.Value as NewValue
from #T as New
  left outer join #T as Old
    on Old.PK = New.PK and
       Old.ColumnName = New.ColumnName and
       Old.Version = New.Version + 1;