Context Navigation

testbed.cpp @ 177:ee6efdda9ac6

Revision 177:ee6efdda9ac6, 57.0 KB (checked in by Andreas Schaefer <gentryx@…>, 14 months ago)
testing hard-coded offsets to simplify address generation logic

Rev	Line
[172]	1	#include <cmath>
	2	#include <typeinfo>
	3	// #include <CL/cl.h>
[6]	4	#include <iostream>
[172]	5	#include <emmintrin.h>
	6	// #include <pmmintrin.h>
	7	#include <sys/time.h>
[6]	8	#include <vector>
[172]	9	#include <libgeodecomp/misc/grid.h>
	10
	11	using namespace LibGeoDecomp;
	12
[174]	13	class Scalar2D
[172]	14	{
	15	public:
[174]	16	static int coefficients()
	17	{
	18	return 0;
	19	}
	20
[172]	21	inline void step(double src, double dst, int offset, int startX, int endX)
	22	{
	23	for (int x = startX; x < endX; ++x) {
	24	dst[x] = (src[x - offset] + src[x - 1] + src[x] + src[x + 1] + src[x + offset]) * 0.2;
	25	}
	26	}
	27
	28	int flops()
	29	{
	30	return 5;
	31	}
	32	};
	33
[174]	34	class VectorizedSSEMelbourneShuffle2D
[172]	35	{
	36	public:
[174]	37	static int coefficients()
	38	{
	39	return 0;
	40	}
	41
[172]	42	inline void step(double src, double dst, int offset, int startX, int endX)
	43	{
	44	int x = startX;
[174]	45	Scalar2D scalarUpdater;
[172]	46
	47	if ((x & 1) == 1) {
	48	scalarUpdater.step(src, dst, offset, x, x + 1);
	49	x += 1;
	50	}
	51
	52	__m128d oneFifth = _mm_set_pd(1.0/3.0, 1.0/3.0);
	53	__m128d buff0 = _mm_loadu_pd(src + x - 1);
	54	__m128d same0 = _mm_load_pd(src + x + 0);
	55
	56	int paddedEndX = endX - 7;
	57	for (; x < paddedEndX; x += 8) {
	58	// load center row
	59	__m128d same1 = _mm_load_pd(src + x + 2);
	60	__m128d same2 = _mm_load_pd(src + x + 4);
	61	__m128d same3 = _mm_load_pd(src + x + 6);
	62	__m128d same4 = _mm_load_pd(src + x + 8);
	63
	64	// shuffle values obtain left/right neighbors
	65	__m128d buff1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
	66	__m128d buff2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
	67	__m128d buff3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
	68	__m128d buff4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
	69
	70	// load top row
	71	__m128d temp0 = _mm_load_pd(src - offset + x + 0);
	72	__m128d temp1 = _mm_load_pd(src - offset + x + 2);
	73	__m128d temp2 = _mm_load_pd(src - offset + x + 4);
	74	__m128d temp3 = _mm_load_pd(src - offset + x + 6);
	75
	76	// add center row with left...
	77	same0 = _mm_add_pd(same0, buff0);
	78	same1 = _mm_add_pd(same1, buff1);
	79	same2 = _mm_add_pd(same2, buff2);
	80	same3 = _mm_add_pd(same3, buff3);
	81
	82	// ...and right neighbors
	83	same0 = _mm_add_pd(same0, buff1);
	84	same1 = _mm_add_pd(same1, buff2);
	85	same2 = _mm_add_pd(same2, buff3);
	86	same3 = _mm_add_pd(same3, buff4);
	87
	88	// load bottom row
	89	buff0 = _mm_load_pd(src + offset + x + 0);
	90	buff1 = _mm_load_pd(src + offset + x + 2);
	91	buff2 = _mm_load_pd(src + offset + x + 4);
	92	buff3 = _mm_load_pd(src + offset + x + 6);
	93
	94	// add top row
	95	same0 = _mm_add_pd(same0, temp0);
	96	same1 = _mm_add_pd(same1, temp1);
	97	same2 = _mm_add_pd(same2, temp2);
	98	same3 = _mm_add_pd(same3, temp3);
	99
	100	// add bottom row
	101	same0 = _mm_add_pd(same0, buff0);
	102	same1 = _mm_add_pd(same1, buff1);
	103	same2 = _mm_add_pd(same2, buff2);
	104	same3 = _mm_add_pd(same3, buff3);
	105
	106	// scale down...
	107	same0 = _mm_mul_pd(same0, oneFifth);
	108	same1 = _mm_mul_pd(same1, oneFifth);
	109	same2 = _mm_mul_pd(same2, oneFifth);
	110	same3 = _mm_mul_pd(same3, oneFifth);
	111
	112	// ...and store
	113	_mm_store_pd(dst + 0, same0);
	114	_mm_store_pd(dst + 2, same1);
	115	_mm_store_pd(dst + 4, same2);
	116	_mm_store_pd(dst + 6, same3);
	117
	118	same0 = same4;
	119	buff0 = buff4;
	120	}
	121
	122	scalarUpdater.step(src, dst, offset, x, endX);
	123	}
	124
	125	int flops()
	126	{
	127	return 5;
	128	}
	129	};
	130
	131
[174]	132	template<typename UPDATER, int DIM>
[172]	133	class Benchmark
	134	{
	135	public:
[174]	136	typedef Grid<double, typename Topologies::Cube<DIM>::Topology> GridType;
	137
	138
	139	void run(Coord<DIM> dim, int repeats)
[172]	140	{
[174]	141	Coord<DIM> coeffDim = dim;
	142	coeffDim.c[DIM - 1] *= UPDATER::coefficients();
	143	GridType coeff(coeffDim, 0.1);
	144	GridType a(dim, 1.0);
	145	GridType b(dim, 1.0);
[172]	146
	147	GridType *oldGrid = &a;
	148	GridType *newGrid = &b;
	149
	150	UPDATER updater;
	151
	152	long long tStart = getUTtime();
[174]	153	std::vector<double*> coefficients(UPDATER::coefficients());
[172]	154
	155	for (int t = 0; t < repeats; ++t) {
[174]	156	for (int z = 1; z < dim.c[2] - 1; ++z) {
	157	for (int y = 1; y < dim.c[1] - 1; ++y) {
	158	Coord<DIM> c(0, y, z);
	159	Coord<DIM> coeffCoord = c;
	160	for (int i = 0; i < UPDATER::coefficients(); ++i) {
	161	coefficients[i] = &coeff.at(coeffCoord);
	162	coeffCoord.c[DIM - 1] += dim.c[DIM - 1];
	163	}
[177]	164	updater.step(
	165	&coefficients[0],
	166	&oldGrid->at(c),
	167	&newGrid->at(c),
	168	dim.c[0],
	169	dim.c[0] * dim.c[1],
	170	1,
	171	dim.c[0] - 1);
[176]	172	// double *source[9] = {
	173	// &oldGrid->at(Coord<DIM>(0, y - 1, z + 1)),
	174	// &oldGrid->at(Coord<DIM>(0, y - 1, z )),
	175	// &oldGrid->at(Coord<DIM>(0, y - 1, z - 1)),
	176	// &oldGrid->at(Coord<DIM>(0, y, z + 1)),
	177	// &oldGrid->at(Coord<DIM>(0, y, z )),
	178	// &oldGrid->at(Coord<DIM>(0, y, z - 1)),
	179	// &oldGrid->at(Coord<DIM>(0, y + 1, z + 1)),
	180	// &oldGrid->at(Coord<DIM>(0, y + 1, z )),
	181	// &oldGrid->at(Coord<DIM>(0, y + 1, z - 1))
	182	// };
	183	// updater.update(source, &newGrid->at(c), 1, dim.c[0] - 1);
[174]	184	}
[172]	185	}
	186	std::swap(newGrid, oldGrid);
	187	}
	188
	189	long long tEnd = getUTtime();
	190	evaluate(dim, repeats, tEnd - tStart);
	191	}
	192
	193	void exercise()
	194	{
	195	std::cout << "# " << typeid(UPDATER).name() << "\n";
	196	int lastDim = 0;
	197	for (int i = 4; i <= 4096; i *= 2) {
	198	int intermediateSteps = 8;
	199	for (int j = 0; j < intermediateSteps; ++j) {
	200	int d = i * std::pow(2, j * (1.0 / intermediateSteps));
	201	if (d % 2) {
	202	d += 1;
	203	}
	204
	205	if (d > lastDim) {
	206	lastDim = d;
[174]	207	Coord<DIM> dim;
[176]	208	dim.c[0] = d;
	209	dim.c[1] = 4;
	210	dim.c[2] = 4;
	211	// for (int i = 0; i < DIM; ++i)
	212	// dim.c[i] = d;
[174]	213	int repeats = std::max(1, 10000000 / dim.prod());
[172]	214	run(dim, repeats);
	215	}
	216	}
	217	}
	218	std::cout << "\n";
	219	}
	220
	221	private:
	222	long long getUTtime()
	223	{
	224	timeval t;
	225	gettimeofday(&t, 0);
	226	return (long long)t.tv_sec * 1000000 + t.tv_usec;
	227	}
	228
[174]	229	void evaluate(Coord<DIM> dim, int repeats, long long uTime)
[172]	230	{
	231	double seconds = 1.0 * uTime / 1000 / 1000;
[174]	232	Coord<DIM> inner = dim;
	233	for (int i = 0; i < DIM; ++i)
	234	inner.c[i] -= 2;
	235	double gflops = 1.0 * UPDATER().flops() * inner.prod() *
[172]	236	repeats / 1000 / 1000 / 1000 / seconds;
[174]	237	std::cout << dim.x() << " " << gflops << "\n";
[172]	238	}
	239
	240
	241	};
[5]	242
[176]	243	#define TN 0
	244	#define T 1
	245	#define TS 2
	246	#define N 3
	247	#define C 4
	248	#define S 5
	249	#define BN 6
	250	#define B 7
	251	#define BS 8
	252
	253	class Jacobi3D
	254	{
	255	public:
	256	static int coefficients()
	257	{
	258	return 1;
	259	}
	260
	261	inline void update(double *src, double dst, int startX, int endX)
	262	{
	263	int x = startX;
	264
	265	if ((x & 1) == 1) {
	266	updateScalar(src, dst, x, x + 1);
	267	x += 1;
	268	}
	269
	270	__m128d oneSeventh = _mm_set_pd(1.0/7.0, 1.0/7.0);
	271	__m128d buff0 = _mm_loadu_pd(src[C] + x - 1);
	272	__m128d same0 = _mm_load_pd(src[C] + x + 0);
	273
	274	int paddedEndX = endX - 7;
	275	for (; x < paddedEndX; x += 8) {
	276	// load center row
	277	__m128d same1 = _mm_load_pd(src[0] + x + 2);
	278	__m128d same2 = _mm_load_pd(src[0] + x + 4);
	279	__m128d same3 = _mm_load_pd(src[0] + x + 6);
	280	__m128d same4 = _mm_load_pd(src[0] + x + 8);
	281
	282	// shuffle values obtain left/right neighbors
	283	__m128d buff1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
	284	__m128d buff2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
	285	__m128d buff3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
	286	__m128d buff4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
	287
	288	// load top row
	289	__m128d temp0 = _mm_load_pd(src[T] + x + 0);
	290	__m128d temp1 = _mm_load_pd(src[T] + x + 2);
	291	__m128d temp2 = _mm_load_pd(src[T] + x + 4);
	292	__m128d temp3 = _mm_load_pd(src[T] + x + 6);
	293
	294	// add center row with left...
	295	same0 = _mm_add_pd(same0, buff0);
	296	same1 = _mm_add_pd(same1, buff1);
	297	same2 = _mm_add_pd(same2, buff2);
	298	same3 = _mm_add_pd(same3, buff3);
	299
	300	// ...and right neighbors
	301	same0 = _mm_add_pd(same0, buff1);
	302	same1 = _mm_add_pd(same1, buff2);
	303	same2 = _mm_add_pd(same2, buff3);
	304	same3 = _mm_add_pd(same3, buff4);
	305
	306	// load bottom row
	307	buff0 = _mm_load_pd(src[B] + x + 0);
	308	buff1 = _mm_load_pd(src[B] + x + 2);
	309	buff2 = _mm_load_pd(src[B] + x + 4);
	310	buff3 = _mm_load_pd(src[B] + x + 6);
	311
	312	// add top row
	313	same0 = _mm_add_pd(same0, temp0);
	314	same1 = _mm_add_pd(same1, temp1);
	315	same2 = _mm_add_pd(same2, temp2);
	316	same3 = _mm_add_pd(same3, temp3);
	317
	318	// load north row
	319	temp0 = _mm_load_pd(src[N] + x + 0);
	320	temp1 = _mm_load_pd(src[N] + x + 2);
	321	temp2 = _mm_load_pd(src[N] + x + 4);
	322	temp3 = _mm_load_pd(src[N] + x + 6);
	323
	324	// add bottom row
	325	same0 = _mm_add_pd(same0, buff0);
	326	same1 = _mm_add_pd(same1, buff1);
	327	same2 = _mm_add_pd(same2, buff2);
	328	same3 = _mm_add_pd(same3, buff3);
	329
	330	// load south row
	331	buff0 = _mm_load_pd(src[S] + x + 0);
	332	buff1 = _mm_load_pd(src[S] + x + 2);
	333	buff2 = _mm_load_pd(src[S] + x + 4);
	334	buff3 = _mm_load_pd(src[S] + x + 6);
	335
	336	// add north row
	337	same0 = _mm_add_pd(same0, temp0);
	338	same1 = _mm_add_pd(same1, temp1);
	339	same2 = _mm_add_pd(same2, temp2);
	340	same3 = _mm_add_pd(same3, temp3);
	341
	342	// add south row
	343	same0 = _mm_add_pd(same0, buff0);
	344	same1 = _mm_add_pd(same1, buff1);
	345	same2 = _mm_add_pd(same2, buff2);
	346	same3 = _mm_add_pd(same3, buff3);
	347
	348	// scale down...
	349	same0 = _mm_mul_pd(same0, oneSeventh);
	350	same1 = _mm_mul_pd(same1, oneSeventh);
	351	same2 = _mm_mul_pd(same2, oneSeventh);
	352	same3 = _mm_mul_pd(same3, oneSeventh);
	353
	354	// ...and store
	355	_mm_store_pd(dst + x + 0, same0);
	356	_mm_store_pd(dst + x + 2, same1);
	357	_mm_store_pd(dst + x + 4, same2);
	358	_mm_store_pd(dst + x + 6, same3);
	359
	360	same0 = same4;
	361	buff0 = buff4;
	362	}
	363
	364	updateScalar(src, dst, x, endX);
	365	}
	366
	367	inline void updateScalar(double src[9], double dst, int startX, int endX)
	368	{
	369	for (int x = startX; x < endX; ++x) {
	370	dst[x] =
	371	(src[S][x] +
	372	src[T][x] +
	373	src[C][x - 1] +
	374	src[C][x] +
	375	src[C][x + 1] +
	376	src[B][x] +
	377	src[N][x]) * (1.0 / 7.0);
	378	}
	379	}
	380
	381	int flops()
	382	{
	383	return 8;
	384	}
	385	};
	386
	387
[174]	388	class Scalar3D
	389	{
	390	public:
	391	static int coefficients()
	392	{
	393	return 7;
	394	}
	395
	396	inline void step(double coeff[7], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
	397	{
	398	for (int x = startX; x < endX; ++x) {
	399	dst[x] =
	400	coeff[0][x] * src[x - offsetZ] +
	401	coeff[1][x] * src[x - offsetY] +
	402	coeff[2][x] * src[x - 1] +
	403	coeff[3][x] * src[x] +
	404	coeff[4][x] * src[x + 1] +
	405	coeff[5][x] * src[x + offsetY] +
	406	coeff[6][x] * src[x + offsetZ];
	407	}
	408	}
	409
	410	int flops()
	411	{
	412	return 13;
	413	}
	414	};
	415
	416	class Vectorized3D
	417	{
	418	public:
	419	static int coefficients()
	420	{
	421	return 7;
	422	}
	423
	424	inline void step(double *coeff, double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
	425	{
	426	int x = startX;
	427	Scalar3D scalarUpdater;
	428
	429	if ((x & 1) == 1) {
	430	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, x + 1);
	431	x += 1;
	432	}
	433
	434	__m128d same0 = _mm_load_pd(src + x + 0);
	435	__m128d neig0 = _mm_loadu_pd(src + x + 1);
	436
	437	int paddedEndX = endX - 7;
	438	for (; x < paddedEndX; x += 8) {
	439	__m128d same1 = _mm_load_pd(src + x + 2);
	440	__m128d same2 = _mm_load_pd(src + x + 4);
	441	__m128d same3 = _mm_load_pd(src + x + 6);
	442	__m128d same4 = _mm_load_pd(src + x + 8);
	443
	444	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
	445	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
	446	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
	447	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
	448
	449	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[3][x + 0]));
	450	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[3][x + 2]));
	451	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[3][x + 4]));
	452	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[3][x + 6]));
	453
	454	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[2][x + 0]));
	455	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[2][x + 2]));
	456	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[2][x + 4]));
	457	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[2][x + 6]));
	458
	459	same0 = _mm_add_pd(same0, temp1);
	460	same1 = _mm_add_pd(same1, temp2);
	461	same2 = _mm_add_pd(same2, temp3);
	462	same3 = _mm_add_pd(same3, temp4);
	463
	464	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[4][x + 0]));
	465	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[4][x + 2]));
	466	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[4][x + 4]));
	467	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[4][x + 6]));
	468
	469	same0 = _mm_add_pd(same0, temp1);
	470	same1 = _mm_add_pd(same1, temp2);
	471	same2 = _mm_add_pd(same2, temp3);
	472	same3 = _mm_add_pd(same3, temp4);
	473
	474	neig0 = _mm_load_pd(src + x - offsetZ + 0);
	475	neig1 = _mm_load_pd(src + x - offsetZ + 2);
	476	neig2 = _mm_load_pd(src + x - offsetZ + 4);
	477	neig3 = _mm_load_pd(src + x - offsetZ + 6);
	478
	479	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][x + 0]));
	480	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][x + 2]));
	481	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][x + 4]));
	482	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][x + 6]));
	483
	484	same0 = _mm_add_pd(same0, temp1);
	485	same1 = _mm_add_pd(same1, temp2);
	486	same2 = _mm_add_pd(same2, temp3);
	487	same3 = _mm_add_pd(same3, temp4);
	488
	489	neig0 = _mm_load_pd(src + x - offsetY + 0);
	490	neig1 = _mm_load_pd(src + x - offsetY + 2);
	491	neig2 = _mm_load_pd(src + x - offsetY + 4);
	492	neig3 = _mm_load_pd(src + x - offsetY + 6);
	493
	494	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[1][x + 0]));
	495	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[1][x + 2]));
	496	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[1][x + 4]));
	497	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[1][x + 6]));
	498
	499	same0 = _mm_add_pd(same0, temp1);
	500	same1 = _mm_add_pd(same1, temp2);
	501	same2 = _mm_add_pd(same2, temp3);
	502	same3 = _mm_add_pd(same3, temp4);
	503
	504	neig0 = _mm_load_pd(src + x + offsetY + 0);
	505	neig1 = _mm_load_pd(src + x + offsetY + 2);
	506	neig2 = _mm_load_pd(src + x + offsetY + 4);
	507	neig3 = _mm_load_pd(src + x + offsetY + 6);
	508
	509	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[5][x + 0]));
	510	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[5][x + 2]));
	511	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[5][x + 4]));
	512	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[5][x + 6]));
	513
	514	same0 = _mm_add_pd(same0, temp1);
	515	same1 = _mm_add_pd(same1, temp2);
	516	same2 = _mm_add_pd(same2, temp3);
	517	same3 = _mm_add_pd(same3, temp4);
	518
	519	neig0 = _mm_load_pd(src + x + offsetZ + 0);
	520	neig1 = _mm_load_pd(src + x + offsetZ + 2);
	521	neig2 = _mm_load_pd(src + x + offsetZ + 4);
	522	neig3 = _mm_load_pd(src + x + offsetZ + 6);
	523
[175]	524	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[6][x + 0]));
	525	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[6][x + 2]));
	526	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[6][x + 4]));
	527	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[6][x + 6]));
[174]	528
	529	same0 = _mm_add_pd(same0, temp1);
	530	same1 = _mm_add_pd(same1, temp2);
	531	same2 = _mm_add_pd(same2, temp3);
	532	same3 = _mm_add_pd(same3, temp4);
	533
	534	_mm_store_pd(dst + 0, same0);
	535	_mm_store_pd(dst + 2, same1);
	536	_mm_store_pd(dst + 4, same2);
	537	_mm_store_pd(dst + 6, same3);
	538
	539	same0 = same4;
	540	neig0 = neig4;
	541
	542	// dst[x] =
	543	// coeff[0][x] * src[x - offsetZ] +
	544	// coeff[1][x] * src[x - offsetY] +
	545	// coeff[2][x] * src[x - 1] +
	546	// coeff[3][x] * src[x] +
	547	// coeff[4][x] * src[x + 1] +
	548	// coeff[5][x] * src[x + offsetY] +
	549	// coeff[6][x] * src[x + offsetZ];
	550	}
	551
	552	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
	553	}
	554
	555	int flops()
	556	{
	557	return 13;
	558	}
	559	};
	560
[175]	561	class ExtendedScalar3D
	562	{
	563	public:
	564	static int coefficients()
	565	{
	566	return 13;
	567	}
	568
	569	inline void step(double coeff[13], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
	570	{
	571	for (int x = startX; x < endX; ++x) {
	572	dst[x] =
	573	coeff[ 0][x] * src[x - offsetZ - offsetY] +
	574	coeff[ 1][x] * src[x - offsetZ] +
	575	coeff[ 2][x] * src[x - offsetZ + offsetY] +
	576	coeff[ 3][x] * src[x - offsetY] +
	577	coeff[ 4][x] * src[x - 1] +
	578	coeff[ 5][x] * src[x] +
	579	coeff[ 6][x] * src[x + 1] +
	580	coeff[ 7][x] * src[x + offsetY] +
	581	coeff[ 8][x] * src[x + offsetZ - offsetY] +
	582	coeff[ 9][x] * src[x + offsetZ] +
	583	coeff[10][x] * src[x + offsetZ + offsetY] +
	584	coeff[11][x - offsetZ - offsetY] +
	585	coeff[11][x - offsetZ] +
	586	coeff[11][x - offsetZ + offsetY] +
	587	coeff[11][x - offsetY] +
	588	coeff[11][x] +
	589	coeff[11][x + offsetY] +
	590	coeff[11][x + offsetZ - offsetY] +
	591	coeff[11][x + offsetZ] +
	592	coeff[11][x + offsetZ + offsetY] +
	593	coeff[12][x - offsetZ - offsetY] +
	594	coeff[12][x - offsetZ] +
	595	coeff[12][x - offsetZ + offsetY] +
	596	coeff[12][x - offsetY] +
	597	coeff[12][x] +
	598	coeff[12][x + offsetY] +
	599	coeff[12][x + offsetZ - offsetY] +
	600	coeff[12][x + offsetZ] +
	601	coeff[12][x + offsetZ + offsetY];
	602	}
	603	}
	604
	605	int flops()
	606	{
	607	return 40;
	608	}
	609	};
	610
	611	class ExtendedVectorized3D
	612	{
	613	public:
	614	static int coefficients()
	615	{
	616	return 13;
	617	}
	618
	619	inline void step(double coeff[13], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
	620	{
	621	int x = startX;
	622	ExtendedScalar3D scalarUpdater;
	623
	624	if ((x & 1) == 1) {
	625	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, x + 1);
	626	x += 1;
	627	}
	628
	629	__m128d same0 = _mm_load_pd(src + x + 0);
	630	__m128d neig0 = _mm_loadu_pd(src + x + 1);
	631
	632	int paddedEndX = endX - 7;
	633	for (; x < paddedEndX; x += 8) {
	634	__m128d same1 = _mm_load_pd(src + x + 2);
	635	__m128d same2 = _mm_load_pd(src + x + 4);
	636	__m128d same3 = _mm_load_pd(src + x + 6);
	637	__m128d same4 = _mm_load_pd(src + x + 8);
	638
	639	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
	640	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
	641	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
	642	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
	643
	644	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[3][x + 0]));
	645	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[3][x + 2]));
	646	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[3][x + 4]));
	647	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[3][x + 6]));
	648
	649	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[2][x + 0]));
	650	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[2][x + 2]));
	651	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[2][x + 4]));
	652	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[2][x + 6]));
	653
	654	same0 = _mm_add_pd(same0, temp1);
	655	same1 = _mm_add_pd(same1, temp2);
	656	same2 = _mm_add_pd(same2, temp3);
	657	same3 = _mm_add_pd(same3, temp4);
	658
	659	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[4][x + 0]));
	660	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[4][x + 2]));
	661	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[4][x + 4]));
	662	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[4][x + 6]));
	663
	664	same0 = _mm_add_pd(same0, temp1);
	665	same1 = _mm_add_pd(same1, temp2);
	666	same2 = _mm_add_pd(same2, temp3);
	667	same3 = _mm_add_pd(same3, temp4);
	668
	669	neig0 = _mm_load_pd(src + x - offsetZ + 0);
	670	neig1 = _mm_load_pd(src + x - offsetZ + 2);
	671	neig2 = _mm_load_pd(src + x - offsetZ + 4);
	672	neig3 = _mm_load_pd(src + x - offsetZ + 6);
	673
	674	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][x + 0]));
	675	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][x + 2]));
	676	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][x + 4]));
	677	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][x + 6]));
	678
	679	same0 = _mm_add_pd(same0, temp1);
	680	same1 = _mm_add_pd(same1, temp2);
	681	same2 = _mm_add_pd(same2, temp3);
	682	same3 = _mm_add_pd(same3, temp4);
	683
	684	neig0 = _mm_load_pd(src + x - offsetY + 0);
	685	neig1 = _mm_load_pd(src + x - offsetY + 2);
	686	neig2 = _mm_load_pd(src + x - offsetY + 4);
	687	neig3 = _mm_load_pd(src + x - offsetY + 6);
	688
	689	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[1][x + 0]));
	690	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[1][x + 2]));
	691	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[1][x + 4]));
	692	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[1][x + 6]));
	693
	694	same0 = _mm_add_pd(same0, temp1);
	695	same1 = _mm_add_pd(same1, temp2);
	696	same2 = _mm_add_pd(same2, temp3);
	697	same3 = _mm_add_pd(same3, temp4);
	698
	699	neig0 = _mm_load_pd(src + x + offsetY + 0);
	700	neig1 = _mm_load_pd(src + x + offsetY + 2);
	701	neig2 = _mm_load_pd(src + x + offsetY + 4);
	702	neig3 = _mm_load_pd(src + x + offsetY + 6);
	703
	704	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[5][x + 0]));
	705	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[5][x + 2]));
	706	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[5][x + 4]));
	707	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[5][x + 6]));
	708
	709	same0 = _mm_add_pd(same0, temp1);
	710	same1 = _mm_add_pd(same1, temp2);
	711	same2 = _mm_add_pd(same2, temp3);
	712	same3 = _mm_add_pd(same3, temp4);
	713
	714	//xxxxxxxxxxxxx
	715	neig0 = _mm_load_pd(src + x + offsetZ + 0);
	716	neig1 = _mm_load_pd(src + x + offsetZ + 2);
	717	neig2 = _mm_load_pd(src + x + offsetZ + 4);
	718	neig3 = _mm_load_pd(src + x + offsetZ + 6);
	719
	720	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[6][x + 0]));
	721	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[6][x + 2]));
	722	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[6][x + 4]));
	723	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[6][x + 6]));
	724
	725	same0 = _mm_add_pd(same0, temp1);
	726	same1 = _mm_add_pd(same1, temp2);
	727	same2 = _mm_add_pd(same2, temp3);
	728	same3 = _mm_add_pd(same3, temp4);
	729
	730	//xxxxxxxxxxxxx
	731	neig0 = _mm_load_pd(src + x - offsetZ - offsetY + 0);
	732	neig1 = _mm_load_pd(src + x - offsetZ - offsetY + 2);
	733	neig2 = _mm_load_pd(src + x - offsetZ - offsetY + 4);
	734	neig3 = _mm_load_pd(src + x - offsetZ - offsetY + 6);
	735
	736	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[7][x + 0]));
	737	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[7][x + 2]));
	738	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[7][x + 4]));
	739	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[7][x + 6]));
	740
	741	same0 = _mm_add_pd(same0, temp1);
	742	same1 = _mm_add_pd(same1, temp2);
	743	same2 = _mm_add_pd(same2, temp3);
	744	same3 = _mm_add_pd(same3, temp4);
	745
	746	//xxxxxxxxxxxxx
	747	neig0 = _mm_load_pd(src + x - offsetZ + offsetY + 0);
	748	neig1 = _mm_load_pd(src + x - offsetZ + offsetY + 2);
	749	neig2 = _mm_load_pd(src + x - offsetZ + offsetY + 4);
	750	neig3 = _mm_load_pd(src + x - offsetZ + offsetY + 6);
	751
	752	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[8][x + 0]));
	753	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[8][x + 2]));
	754	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[8][x + 4]));
	755	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[8][x + 6]));
	756
	757	same0 = _mm_add_pd(same0, temp1);
	758	same1 = _mm_add_pd(same1, temp2);
	759	same2 = _mm_add_pd(same2, temp3);
	760	same3 = _mm_add_pd(same3, temp4);
	761
	762	//xxxxxxxxxxxxx
	763	neig0 = _mm_load_pd(src + x + offsetZ - offsetY + 0);
	764	neig1 = _mm_load_pd(src + x + offsetZ - offsetY + 2);
	765	neig2 = _mm_load_pd(src + x + offsetZ - offsetY + 4);
	766	neig3 = _mm_load_pd(src + x + offsetZ - offsetY + 6);
	767
	768	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[9][x + 0]));
	769	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[9][x + 2]));
	770	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[9][x + 4]));
	771	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[9][x + 6]));
	772
	773	same0 = _mm_add_pd(same0, temp1);
	774	same1 = _mm_add_pd(same1, temp2);
	775	same2 = _mm_add_pd(same2, temp3);
	776	same3 = _mm_add_pd(same3, temp4);
	777
	778	//xxxxxxxxxxxxx
	779	neig0 = _mm_load_pd(src + x + offsetZ + offsetY + 0);
	780	neig1 = _mm_load_pd(src + x + offsetZ + offsetY + 2);
	781	neig2 = _mm_load_pd(src + x + offsetZ + offsetY + 4);
	782	neig3 = _mm_load_pd(src + x + offsetZ + offsetY + 6);
	783
	784	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[10][x + 0]));
	785	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[10][x + 2]));
	786	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[10][x + 4]));
	787	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[10][x + 6]));
	788
	789	same0 = _mm_add_pd(same0, temp1);
	790	same1 = _mm_add_pd(same1, temp2);
	791	same2 = _mm_add_pd(same2, temp3);
	792	same3 = _mm_add_pd(same3, temp4);
	793
	794	//xxxxxxxxxxxxx
	795	neig0 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 0);
	796	neig1 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 2);
	797	neig2 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 4);
	798	neig3 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 6);
	799
	800	same0 = _mm_add_pd(same0, neig0);
	801	same1 = _mm_add_pd(same1, neig1);
	802	same2 = _mm_add_pd(same2, neig2);
	803	same3 = _mm_add_pd(same3, neig3);
	804
	805	//xxxxxxxxxxxxx
	806	neig0 = _mm_load_pd(coeff[11] + x - offsetZ + 0);
	807	neig1 = _mm_load_pd(coeff[11] + x - offsetZ + 2);
	808	neig2 = _mm_load_pd(coeff[11] + x - offsetZ + 4);
	809	neig3 = _mm_load_pd(coeff[11] + x - offsetZ + 6);
	810
	811	same0 = _mm_add_pd(same0, neig0);
	812	same1 = _mm_add_pd(same1, neig1);
	813	same2 = _mm_add_pd(same2, neig2);
	814	same3 = _mm_add_pd(same3, neig3);
	815
	816	//xxxxxxxxxxxxx
	817	neig0 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 0);
	818	neig1 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 2);
	819	neig2 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 4);
	820	neig3 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 6);
	821
	822	same0 = _mm_add_pd(same0, neig0);
	823	same1 = _mm_add_pd(same1, neig1);
	824	same2 = _mm_add_pd(same2, neig2);
	825	same3 = _mm_add_pd(same3, neig3);
	826
	827	//xxxxxxxxxxxxx
	828	neig0 = _mm_load_pd(coeff[11] + x - offsetY + 0);
	829	neig1 = _mm_load_pd(coeff[11] + x - offsetY + 2);
	830	neig2 = _mm_load_pd(coeff[11] + x - offsetY + 4);
	831	neig3 = _mm_load_pd(coeff[11] + x - offsetY + 6);
	832
	833	same0 = _mm_add_pd(same0, neig0);
	834	same1 = _mm_add_pd(same1, neig1);
	835	same2 = _mm_add_pd(same2, neig2);
	836	same3 = _mm_add_pd(same3, neig3);
	837
	838	//xxxxxxxxxxxxx
	839	neig0 = _mm_load_pd(coeff[11] + offsetY + x + 0);
	840	neig1 = _mm_load_pd(coeff[11] + offsetY + x + 2);
	841	neig2 = _mm_load_pd(coeff[11] + offsetY + x + 4);
	842	neig3 = _mm_load_pd(coeff[11] + offsetY + x + 6);
	843
	844	same0 = _mm_add_pd(same0, neig0);
	845	same1 = _mm_add_pd(same1, neig1);
	846	same2 = _mm_add_pd(same2, neig2);
	847	same3 = _mm_add_pd(same3, neig3);
	848
	849	//xxxxxxxxxxxxx
	850	neig0 = _mm_load_pd(coeff[11] + x + 0);
	851	neig1 = _mm_load_pd(coeff[11] + x + 2);
	852	neig2 = _mm_load_pd(coeff[11] + x + 4);
	853	neig3 = _mm_load_pd(coeff[11] + x + 6);
	854
	855	same0 = _mm_add_pd(same0, neig0);
	856	same1 = _mm_add_pd(same1, neig1);
	857	same2 = _mm_add_pd(same2, neig2);
	858	same3 = _mm_add_pd(same3, neig3);
	859
	860	//xxxxxxxxxxxxx
	861	neig0 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 0);
	862	neig1 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 2);
	863	neig2 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 4);
	864	neig3 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 6);
	865
	866	same0 = _mm_add_pd(same0, neig0);
	867	same1 = _mm_add_pd(same1, neig1);
	868	same2 = _mm_add_pd(same2, neig2);
	869	same3 = _mm_add_pd(same3, neig3);
	870
	871	//xxxxxxxxxxxxx
	872	neig0 = _mm_load_pd(coeff[11] + x + offsetZ + 0);
	873	neig1 = _mm_load_pd(coeff[11] + x + offsetZ + 2);
	874	neig2 = _mm_load_pd(coeff[11] + x + offsetZ + 4);
	875	neig3 = _mm_load_pd(coeff[11] + x + offsetZ + 6);
	876
	877	same0 = _mm_add_pd(same0, neig0);
	878	same1 = _mm_add_pd(same1, neig1);
	879	same2 = _mm_add_pd(same2, neig2);
	880	same3 = _mm_add_pd(same3, neig3);
	881
	882	//xxxxxxxxxxxxx
	883	neig0 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 0);
	884	neig1 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 2);
	885	neig2 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 4);
	886	neig3 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 6);
	887
	888	same0 = _mm_add_pd(same0, neig0);
	889	same1 = _mm_add_pd(same1, neig1);
	890	same2 = _mm_add_pd(same2, neig2);
	891	same3 = _mm_add_pd(same3, neig3);
	892
	893	//xxxxxxxxxxxxx
	894	neig0 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 0);
	895	neig1 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 2);
	896	neig2 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 4);
	897	neig3 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 6);
	898
	899	same0 = _mm_add_pd(same0, neig0);
	900	same1 = _mm_add_pd(same1, neig1);
	901	same2 = _mm_add_pd(same2, neig2);
	902	same3 = _mm_add_pd(same3, neig3);
	903
	904	//xxxxxxxxxxxxx
	905	neig0 = _mm_load_pd(coeff[12] + x - offsetZ + 0);
	906	neig1 = _mm_load_pd(coeff[12] + x - offsetZ + 2);
	907	neig2 = _mm_load_pd(coeff[12] + x - offsetZ + 4);
	908	neig3 = _mm_load_pd(coeff[12] + x - offsetZ + 6);
	909
	910	same0 = _mm_add_pd(same0, neig0);
	911	same1 = _mm_add_pd(same1, neig1);
	912	same2 = _mm_add_pd(same2, neig2);
	913	same3 = _mm_add_pd(same3, neig3);
	914
	915	//xxxxxxxxxxxxx
	916	neig0 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 0);
	917	neig1 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 2);
	918	neig2 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 4);
	919	neig3 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 6);
	920
	921	same0 = _mm_add_pd(same0, neig0);
	922	same1 = _mm_add_pd(same1, neig1);
	923	same2 = _mm_add_pd(same2, neig2);
	924	same3 = _mm_add_pd(same3, neig3);
	925
	926	//xxxxxxxxxxxxx
	927	neig0 = _mm_load_pd(coeff[12] + x - offsetY + 0);
	928	neig1 = _mm_load_pd(coeff[12] + x - offsetY + 2);
	929	neig2 = _mm_load_pd(coeff[12] + x - offsetY + 4);
	930	neig3 = _mm_load_pd(coeff[12] + x - offsetY + 6);
	931
	932	same0 = _mm_add_pd(same0, neig0);
	933	same1 = _mm_add_pd(same1, neig1);
	934	same2 = _mm_add_pd(same2, neig2);
	935	same3 = _mm_add_pd(same3, neig3);
	936
	937	//xxxxxxxxxxxxx
	938	neig0 = _mm_load_pd(coeff[12] + offsetY + x + 0);
	939	neig1 = _mm_load_pd(coeff[12] + offsetY + x + 2);
	940	neig2 = _mm_load_pd(coeff[12] + offsetY + x + 4);
	941	neig3 = _mm_load_pd(coeff[12] + offsetY + x + 6);
	942
	943	same0 = _mm_add_pd(same0, neig0);
	944	same1 = _mm_add_pd(same1, neig1);
	945	same2 = _mm_add_pd(same2, neig2);
	946	same3 = _mm_add_pd(same3, neig3);
	947
	948	//xxxxxxxxxxxxx
	949	neig0 = _mm_load_pd(coeff[12] + x + 0);
	950	neig1 = _mm_load_pd(coeff[12] + x + 2);
	951	neig2 = _mm_load_pd(coeff[12] + x + 4);
	952	neig3 = _mm_load_pd(coeff[12] + x + 6);
	953
	954	same0 = _mm_add_pd(same0, neig0);
	955	same1 = _mm_add_pd(same1, neig1);
	956	same2 = _mm_add_pd(same2, neig2);
	957	same3 = _mm_add_pd(same3, neig3);
	958
	959	//xxxxxxxxxxxxx
	960	neig0 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 0);
	961	neig1 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 2);
	962	neig2 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 4);
	963	neig3 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 6);
	964
	965	same0 = _mm_add_pd(same0, neig0);
	966	same1 = _mm_add_pd(same1, neig1);
	967	same2 = _mm_add_pd(same2, neig2);
	968	same3 = _mm_add_pd(same3, neig3);
	969
	970	//xxxxxxxxxxxxx
	971	neig0 = _mm_load_pd(coeff[12] + x + offsetZ + 0);
	972	neig1 = _mm_load_pd(coeff[12] + x + offsetZ + 2);
	973	neig2 = _mm_load_pd(coeff[12] + x + offsetZ + 4);
	974	neig3 = _mm_load_pd(coeff[12] + x + offsetZ + 6);
	975
	976	same0 = _mm_add_pd(same0, neig0);
	977	same1 = _mm_add_pd(same1, neig1);
	978	same2 = _mm_add_pd(same2, neig2);
	979	same3 = _mm_add_pd(same3, neig3);
	980
	981	//xxxxxxxxxxxxx
	982	neig0 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 0);
	983	neig1 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 2);
	984	neig2 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 4);
	985	neig3 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 6);
	986
	987	same0 = _mm_add_pd(same0, neig0);
	988	same1 = _mm_add_pd(same1, neig1);
	989	same2 = _mm_add_pd(same2, neig2);
	990	same3 = _mm_add_pd(same3, neig3);
	991
	992	//yyyyyyyyyyyyy
[177]	993	_mm_store_pd(dst + x + 0, same0);
	994	_mm_store_pd(dst + x + 2, same1);
	995	_mm_store_pd(dst + x + 4, same2);
	996	_mm_store_pd(dst + x + 6, same3);
[175]	997
	998	same0 = same4;
	999	neig0 = neig4;
	1000
	1001	// dst[x] =
	1002	// coeff[0][x] * src[x - offsetZ] +
	1003	// coeff[1][x] * src[x - offsetY] +
	1004	// coeff[2][x] * src[x - 1] +
	1005	// coeff[3][x] * src[x] +
	1006	// coeff[4][x] * src[x + 1] +
	1007	// coeff[5][x] * src[x + offsetY] +
	1008	// coeff[6][x] * src[x + offsetZ];
	1009	}
	1010
	1011	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
	1012	}
	1013
	1014	int flops()
	1015	{
	1016	return 40;
	1017	}
	1018	};
	1019
[177]	1020	template<int DIM_X, int DIM_Y, int DIM_Z>
	1021	class ExtendedVectorized3DFixed
	1022	{
	1023	public:
	1024	static int coefficients()
	1025	{
	1026	return 13;
	1027	}
	1028
	1029	inline void step(double coeff[13], double src, double *dst, int unusedOffsetY, int unusedOffsetZ, int startX, int endX)
	1030	{
	1031	const int SLICE_SIZE = DIM_X * DIM_Y;
	1032	const int TOTAL_SIZE = DIM_X * DIM_Y * DIM_Z;
	1033	const int offsetY = DIM_X;
	1034	const int offsetZ = SLICE_SIZE;
	1035
	1036	int x = startX;
	1037	ExtendedScalar3D scalarUpdater;
	1038
	1039	if ((x & 1) == 1) {
	1040	scalarUpdater.step(coeff, src, dst, DIM_Y, offsetZ, x, x + 1);
	1041	x += 1;
	1042	}
	1043
	1044	__m128d same0 = _mm_load_pd(src + x + 0);
	1045	__m128d neig0 = _mm_loadu_pd(src + x + 1);
	1046
	1047	int paddedEndX = endX - 7;
	1048	for (; x < paddedEndX; x += 8) {
	1049	__m128d same1 = _mm_load_pd(src + x + 2);
	1050	__m128d same2 = _mm_load_pd(src + x + 4);
	1051	__m128d same3 = _mm_load_pd(src + x + 6);
	1052	__m128d same4 = _mm_load_pd(src + x + 8);
	1053
	1054	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
	1055	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
	1056	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
	1057	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
	1058
	1059	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 0]));
	1060	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 2]));
	1061	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 4]));
	1062	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 6]));
	1063
	1064	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 0]));
	1065	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 2]));
	1066	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 4]));
	1067	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 6]));
	1068
	1069	same0 = _mm_add_pd(same0, temp1);
	1070	same1 = _mm_add_pd(same1, temp2);
	1071	same2 = _mm_add_pd(same2, temp3);
	1072	same3 = _mm_add_pd(same3, temp4);
	1073
	1074	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 0]));
	1075	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 2]));
	1076	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 4]));
	1077	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 6]));
	1078
	1079	same0 = _mm_add_pd(same0, temp1);
	1080	same1 = _mm_add_pd(same1, temp2);
	1081	same2 = _mm_add_pd(same2, temp3);
	1082	same3 = _mm_add_pd(same3, temp4);
	1083
	1084	neig0 = _mm_load_pd(src + x - offsetZ + 0);
	1085	neig1 = _mm_load_pd(src + x - offsetZ + 2);
	1086	neig2 = _mm_load_pd(src + x - offsetZ + 4);
	1087	neig3 = _mm_load_pd(src + x - offsetZ + 6);
	1088
	1089	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 0]));
	1090	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 2]));
	1091	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 4]));
	1092	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 6]));
	1093
	1094	same0 = _mm_add_pd(same0, temp1);
	1095	same1 = _mm_add_pd(same1, temp2);
	1096	same2 = _mm_add_pd(same2, temp3);
	1097	same3 = _mm_add_pd(same3, temp4);
	1098
	1099	neig0 = _mm_load_pd(src + x - offsetY + 0);
	1100	neig1 = _mm_load_pd(src + x - offsetY + 2);
	1101	neig2 = _mm_load_pd(src + x - offsetY + 4);
	1102	neig3 = _mm_load_pd(src + x - offsetY + 6);
	1103
	1104	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 0]));
	1105	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 2]));
	1106	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 4]));
	1107	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 6]));
	1108
	1109	same0 = _mm_add_pd(same0, temp1);
	1110	same1 = _mm_add_pd(same1, temp2);
	1111	same2 = _mm_add_pd(same2, temp3);
	1112	same3 = _mm_add_pd(same3, temp4);
	1113
	1114	neig0 = _mm_load_pd(src + x + offsetY + 0);
	1115	neig1 = _mm_load_pd(src + x + offsetY + 2);
	1116	neig2 = _mm_load_pd(src + x + offsetY + 4);
	1117	neig3 = _mm_load_pd(src + x + offsetY + 6);
	1118
	1119	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 0]));
	1120	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 2]));
	1121	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 4]));
	1122	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 6]));
	1123
	1124	same0 = _mm_add_pd(same0, temp1);
	1125	same1 = _mm_add_pd(same1, temp2);
	1126	same2 = _mm_add_pd(same2, temp3);
	1127	same3 = _mm_add_pd(same3, temp4);
	1128
	1129	//xxxxxxxxxxxxx
	1130	neig0 = _mm_load_pd(src + x + offsetZ + 0);
	1131	neig1 = _mm_load_pd(src + x + offsetZ + 2);
	1132	neig2 = _mm_load_pd(src + x + offsetZ + 4);
	1133	neig3 = _mm_load_pd(src + x + offsetZ + 6);
	1134
	1135	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 0]));
	1136	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 2]));
	1137	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 4]));
	1138	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 6]));
	1139
	1140	same0 = _mm_add_pd(same0, temp1);
	1141	same1 = _mm_add_pd(same1, temp2);
	1142	same2 = _mm_add_pd(same2, temp3);
	1143	same3 = _mm_add_pd(same3, temp4);
	1144
	1145	//xxxxxxxxxxxxx
	1146	neig0 = _mm_load_pd(src + x - offsetZ - offsetY + 0);
	1147	neig1 = _mm_load_pd(src + x - offsetZ - offsetY + 2);
	1148	neig2 = _mm_load_pd(src + x - offsetZ - offsetY + 4);
	1149	neig3 = _mm_load_pd(src + x - offsetZ - offsetY + 6);
	1150
	1151	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 0]));
	1152	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 2]));
	1153	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 4]));
	1154	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 6]));
	1155
	1156	same0 = _mm_add_pd(same0, temp1);
	1157	same1 = _mm_add_pd(same1, temp2);
	1158	same2 = _mm_add_pd(same2, temp3);
	1159	same3 = _mm_add_pd(same3, temp4);
	1160
	1161	//xxxxxxxxxxxxx
	1162	neig0 = _mm_load_pd(src + x - offsetZ + offsetY + 0);
	1163	neig1 = _mm_load_pd(src + x - offsetZ + offsetY + 2);
	1164	neig2 = _mm_load_pd(src + x - offsetZ + offsetY + 4);
	1165	neig3 = _mm_load_pd(src + x - offsetZ + offsetY + 6);
	1166
	1167	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 0]));
	1168	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 2]));
	1169	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 4]));
	1170	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 6]));
	1171
	1172	same0 = _mm_add_pd(same0, temp1);
	1173	same1 = _mm_add_pd(same1, temp2);
	1174	same2 = _mm_add_pd(same2, temp3);
	1175	same3 = _mm_add_pd(same3, temp4);
	1176
	1177	//xxxxxxxxxxxxx
	1178	neig0 = _mm_load_pd(src + x + offsetZ - offsetY + 0);
	1179	neig1 = _mm_load_pd(src + x + offsetZ - offsetY + 2);
	1180	neig2 = _mm_load_pd(src + x + offsetZ - offsetY + 4);
	1181	neig3 = _mm_load_pd(src + x + offsetZ - offsetY + 6);
	1182
	1183	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 0]));
	1184	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 2]));
	1185	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 4]));
	1186	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 6]));
	1187
	1188	same0 = _mm_add_pd(same0, temp1);
	1189	same1 = _mm_add_pd(same1, temp2);
	1190	same2 = _mm_add_pd(same2, temp3);
	1191	same3 = _mm_add_pd(same3, temp4);
	1192
	1193	//xxxxxxxxxxxxx
	1194	neig0 = _mm_load_pd(src + x + offsetZ + offsetY + 0);
	1195	neig1 = _mm_load_pd(src + x + offsetZ + offsetY + 2);
	1196	neig2 = _mm_load_pd(src + x + offsetZ + offsetY + 4);
	1197	neig3 = _mm_load_pd(src + x + offsetZ + offsetY + 6);
	1198
	1199	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 0]));
	1200	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 2]));
	1201	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 4]));
	1202	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 6]));
	1203
	1204	same0 = _mm_add_pd(same0, temp1);
	1205	same1 = _mm_add_pd(same1, temp2);
	1206	same2 = _mm_add_pd(same2, temp3);
	1207	same3 = _mm_add_pd(same3, temp4);
	1208
	1209	//xxxxxxxxxxxxx
	1210	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 0);
	1211	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 2);
	1212	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 4);
	1213	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 6);
	1214
	1215	same0 = _mm_add_pd(same0, neig0);
	1216	same1 = _mm_add_pd(same1, neig1);
	1217	same2 = _mm_add_pd(same2, neig2);
	1218	same3 = _mm_add_pd(same3, neig3);
	1219
	1220	//xxxxxxxxxxxxx
	1221	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 0);
	1222	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 2);
	1223	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 4);
	1224	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 6);
	1225
	1226	same0 = _mm_add_pd(same0, neig0);
	1227	same1 = _mm_add_pd(same1, neig1);
	1228	same2 = _mm_add_pd(same2, neig2);
	1229	same3 = _mm_add_pd(same3, neig3);
	1230
	1231	//xxxxxxxxxxxxx
	1232	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 0);
	1233	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 2);
	1234	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 4);
	1235	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 6);
	1236
	1237	same0 = _mm_add_pd(same0, neig0);
	1238	same1 = _mm_add_pd(same1, neig1);
	1239	same2 = _mm_add_pd(same2, neig2);
	1240	same3 = _mm_add_pd(same3, neig3);
	1241
	1242	//xxxxxxxxxxxxx
	1243	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 0);
	1244	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 2);
	1245	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 4);
	1246	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 6);
	1247
	1248	same0 = _mm_add_pd(same0, neig0);
	1249	same1 = _mm_add_pd(same1, neig1);
	1250	same2 = _mm_add_pd(same2, neig2);
	1251	same3 = _mm_add_pd(same3, neig3);
	1252
	1253	//xxxxxxxxxxxxx
	1254	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 0);
	1255	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 2);
	1256	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 4);
	1257	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 6);
	1258
	1259	same0 = _mm_add_pd(same0, neig0);
	1260	same1 = _mm_add_pd(same1, neig1);
	1261	same2 = _mm_add_pd(same2, neig2);
	1262	same3 = _mm_add_pd(same3, neig3);
	1263
	1264	//xxxxxxxxxxxxx
	1265	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 0);
	1266	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 2);
	1267	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 4);
	1268	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 6);
	1269
	1270	same0 = _mm_add_pd(same0, neig0);
	1271	same1 = _mm_add_pd(same1, neig1);
	1272	same2 = _mm_add_pd(same2, neig2);
	1273	same3 = _mm_add_pd(same3, neig3);
	1274
	1275	//xxxxxxxxxxxxx
	1276	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 0);
	1277	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 2);
	1278	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 4);
	1279	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 6);
	1280
	1281	same0 = _mm_add_pd(same0, neig0);
	1282	same1 = _mm_add_pd(same1, neig1);
	1283	same2 = _mm_add_pd(same2, neig2);
	1284	same3 = _mm_add_pd(same3, neig3);
	1285
	1286	//xxxxxxxxxxxxx
	1287	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 0);
	1288	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 2);
	1289	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 4);
	1290	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 6);
	1291
	1292	same0 = _mm_add_pd(same0, neig0);
	1293	same1 = _mm_add_pd(same1, neig1);
	1294	same2 = _mm_add_pd(same2, neig2);
	1295	same3 = _mm_add_pd(same3, neig3);
	1296
	1297	//xxxxxxxxxxxxx
	1298	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 0);
	1299	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 2);
	1300	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 4);
	1301	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 6);
	1302
	1303	same0 = _mm_add_pd(same0, neig0);
	1304	same1 = _mm_add_pd(same1, neig1);
	1305	same2 = _mm_add_pd(same2, neig2);
	1306	same3 = _mm_add_pd(same3, neig3);
	1307
	1308	//xxxxxxxxxxxxx
	1309	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 0);
	1310	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 2);
	1311	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 4);
	1312	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 6);
	1313
	1314	same0 = _mm_add_pd(same0, neig0);
	1315	same1 = _mm_add_pd(same1, neig1);
	1316	same2 = _mm_add_pd(same2, neig2);
	1317	same3 = _mm_add_pd(same3, neig3);
	1318
	1319	//xxxxxxxxxxxxx
	1320	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 0);
	1321	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 2);
	1322	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 4);
	1323	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 6);
	1324
	1325	same0 = _mm_add_pd(same0, neig0);
	1326	same1 = _mm_add_pd(same1, neig1);
	1327	same2 = _mm_add_pd(same2, neig2);
	1328	same3 = _mm_add_pd(same3, neig3);
	1329
	1330	//xxxxxxxxxxxxx
	1331	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 0);
	1332	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 2);
	1333	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 4);
	1334	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 6);
	1335
	1336	same0 = _mm_add_pd(same0, neig0);
	1337	same1 = _mm_add_pd(same1, neig1);
	1338	same2 = _mm_add_pd(same2, neig2);
	1339	same3 = _mm_add_pd(same3, neig3);
	1340
	1341	//xxxxxxxxxxxxx
	1342	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 0);
	1343	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 2);
	1344	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 4);
	1345	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 6);
	1346
	1347	same0 = _mm_add_pd(same0, neig0);
	1348	same1 = _mm_add_pd(same1, neig1);
	1349	same2 = _mm_add_pd(same2, neig2);
	1350	same3 = _mm_add_pd(same3, neig3);
	1351
	1352	//xxxxxxxxxxxxx
	1353	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 0);
	1354	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 2);
	1355	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 4);
	1356	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 6);
	1357
	1358	same0 = _mm_add_pd(same0, neig0);
	1359	same1 = _mm_add_pd(same1, neig1);
	1360	same2 = _mm_add_pd(same2, neig2);
	1361	same3 = _mm_add_pd(same3, neig3);
	1362
	1363	//xxxxxxxxxxxxx
	1364	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 0);
	1365	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 2);
	1366	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 4);
	1367	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 6);
	1368
	1369	same0 = _mm_add_pd(same0, neig0);
	1370	same1 = _mm_add_pd(same1, neig1);
	1371	same2 = _mm_add_pd(same2, neig2);
	1372	same3 = _mm_add_pd(same3, neig3);
	1373
	1374	//xxxxxxxxxxxxx
	1375	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 0);
	1376	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 2);
	1377	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 4);
	1378	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 6);
	1379
	1380	same0 = _mm_add_pd(same0, neig0);
	1381	same1 = _mm_add_pd(same1, neig1);
	1382	same2 = _mm_add_pd(same2, neig2);
	1383	same3 = _mm_add_pd(same3, neig3);
	1384
	1385	//xxxxxxxxxxxxx
	1386	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 0);
	1387	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 2);
	1388	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 4);
	1389	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 6);
	1390
	1391	same0 = _mm_add_pd(same0, neig0);
	1392	same1 = _mm_add_pd(same1, neig1);
	1393	same2 = _mm_add_pd(same2, neig2);
	1394	same3 = _mm_add_pd(same3, neig3);
	1395
	1396	//xxxxxxxxxxxxx
	1397	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 0);
	1398	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 2);
	1399	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 4);
	1400	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 6);
	1401
	1402	same0 = _mm_add_pd(same0, neig0);
	1403	same1 = _mm_add_pd(same1, neig1);
	1404	same2 = _mm_add_pd(same2, neig2);
	1405	same3 = _mm_add_pd(same3, neig3);
	1406
	1407	//yyyyyyyyyyyyy
	1408	_mm_store_pd(dst + x + 0, same0);
	1409	_mm_store_pd(dst + x + 2, same1);
	1410	_mm_store_pd(dst + x + 4, same2);
	1411	_mm_store_pd(dst + x + 6, same3);
	1412
	1413	same0 = same4;
	1414	neig0 = neig4;
	1415
	1416	// dst[x] =
	1417	// coeff[0][x] * src[x - offsetZ] +
	1418	// coeff[1][x] * src[x - offsetY] +
	1419	// coeff[2][x] * src[x - 1] +
	1420	// coeff[3][x] * src[x] +
	1421	// coeff[4][x] * src[x + 1] +
	1422	// coeff[5][x] * src[x + offsetY] +
	1423	// coeff[6][x] * src[x + offsetZ];
	1424	}
	1425
	1426	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
	1427	}
	1428
	1429	int flops()
	1430	{
	1431	return 40;
	1432	}
	1433	};
	1434
[174]	1435
[5]	1436	int main(int argc, char *argv[])
	1437	{
[174]	1438	// Benchmark<Scalar3D, 3>().exercise();
[175]	1439	// Benchmark<Vectorized3D, 3>().exercise();
[174]	1440	// Benchmark<VectorizedSSEMelbourneShuffle2D>().exercise();
[175]	1441	Benchmark<ExtendedVectorized3D, 3>().exercise();
[177]	1442	// Benchmark<ExtendedVectorized3DFixed, 3>().exercise();
[176]	1443	// Benchmark<Jacobi3D, 3>().exercise();
[172]	1444
	1445
[19]	1446	// std::vector<cl::Platform> platforms;
	1447	// cl::Platform::get(&platforms);
[6]	1448
[19]	1449	// for (int i = 0; i < platforms.size(); ++i) {
	1450	// std::string str;
	1451	// platforms[i].getInfo(CL_PLATFORM_NAME, &str);
	1452	// std::cout << "Platform[" << i << "] = " << str << std::endl;
	1453	// }
[6]	1454
[5]	1455	return 0;
	1456	}

Note: See TracBrowser for help on using the browser.

Context Navigation

root/src/testbed/testbed.cpp @ 177:ee6efdda9ac6

Download in other formats: