Context Navigation

testbed.cpp @ 178:cc86edb738ca

Revision 178:cc86edb738ca, 79.8 KB (checked in by Andreas Schaefer <gentryx@…>, 14 months ago)
added next generation interface prototype

Line
1	#include <cmath>
2	#include <typeinfo>
3	// #include <CL/cl.h>
4	#include <iostream>
5	#include <emmintrin.h>
6	// #include <pmmintrin.h>
7	#include <sys/time.h>
8	#include <vector>
9	#include <libgeodecomp/misc/grid.h>
10
11	using namespace LibGeoDecomp;
12
13	class Scalar2D
14	{
15	public:
16	static int coefficients()
17	{
18	return 0;
19	}
20
21	inline void step(double src, double dst, int offset, int startX, int endX)
22	{
23	for (int x = startX; x < endX; ++x) {
24	dst[x] = (src[x - offset] + src[x - 1] + src[x] + src[x + 1] + src[x + offset]) * 0.2;
25	}
26	}
27
28	int flops()
29	{
30	return 5;
31	}
32	};
33
34	class VectorizedSSEMelbourneShuffle2D
35	{
36	public:
37	static int coefficients()
38	{
39	return 0;
40	}
41
42	inline void step(double src, double dst, int offset, int startX, int endX)
43	{
44	int x = startX;
45	Scalar2D scalarUpdater;
46
47	if ((x & 1) == 1) {
48	scalarUpdater.step(src, dst, offset, x, x + 1);
49	x += 1;
50	}
51
52	__m128d oneFifth = _mm_set_pd(1.0/3.0, 1.0/3.0);
53	__m128d buff0 = _mm_loadu_pd(src + x - 1);
54	__m128d same0 = _mm_load_pd(src + x + 0);
55
56	int paddedEndX = endX - 7;
57	for (; x < paddedEndX; x += 8) {
58	// load center row
59	__m128d same1 = _mm_load_pd(src + x + 2);
60	__m128d same2 = _mm_load_pd(src + x + 4);
61	__m128d same3 = _mm_load_pd(src + x + 6);
62	__m128d same4 = _mm_load_pd(src + x + 8);
63
64	// shuffle values obtain left/right neighbors
65	__m128d buff1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
66	__m128d buff2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
67	__m128d buff3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
68	__m128d buff4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
69
70	// load top row
71	__m128d temp0 = _mm_load_pd(src - offset + x + 0);
72	__m128d temp1 = _mm_load_pd(src - offset + x + 2);
73	__m128d temp2 = _mm_load_pd(src - offset + x + 4);
74	__m128d temp3 = _mm_load_pd(src - offset + x + 6);
75
76	// add center row with left...
77	same0 = _mm_add_pd(same0, buff0);
78	same1 = _mm_add_pd(same1, buff1);
79	same2 = _mm_add_pd(same2, buff2);
80	same3 = _mm_add_pd(same3, buff3);
81
82	// ...and right neighbors
83	same0 = _mm_add_pd(same0, buff1);
84	same1 = _mm_add_pd(same1, buff2);
85	same2 = _mm_add_pd(same2, buff3);
86	same3 = _mm_add_pd(same3, buff4);
87
88	// load bottom row
89	buff0 = _mm_load_pd(src + offset + x + 0);
90	buff1 = _mm_load_pd(src + offset + x + 2);
91	buff2 = _mm_load_pd(src + offset + x + 4);
92	buff3 = _mm_load_pd(src + offset + x + 6);
93
94	// add top row
95	same0 = _mm_add_pd(same0, temp0);
96	same1 = _mm_add_pd(same1, temp1);
97	same2 = _mm_add_pd(same2, temp2);
98	same3 = _mm_add_pd(same3, temp3);
99
100	// add bottom row
101	same0 = _mm_add_pd(same0, buff0);
102	same1 = _mm_add_pd(same1, buff1);
103	same2 = _mm_add_pd(same2, buff2);
104	same3 = _mm_add_pd(same3, buff3);
105
106	// scale down...
107	same0 = _mm_mul_pd(same0, oneFifth);
108	same1 = _mm_mul_pd(same1, oneFifth);
109	same2 = _mm_mul_pd(same2, oneFifth);
110	same3 = _mm_mul_pd(same3, oneFifth);
111
112	// ...and store
113	_mm_store_pd(dst + 0, same0);
114	_mm_store_pd(dst + 2, same1);
115	_mm_store_pd(dst + 4, same2);
116	_mm_store_pd(dst + 6, same3);
117
118	same0 = same4;
119	buff0 = buff4;
120	}
121
122	scalarUpdater.step(src, dst, offset, x, endX);
123	}
124
125	int flops()
126	{
127	return 5;
128	}
129	};
130
131	template<int DIM_X, int DIM_Y, int DIM_Z>
132	class Neighborhood
133	{
134	public:
135	Neighborhood(double _coefficients, double _source) :
136	coefficients(_coefficients),
137	source(_source)
138	{}
139
140	template<int X, int Y, int Z, int C>
141	const double& coeff(const int& x) const
142	{
143	return coefficients[C * DIM_X * DIM_Y * DIM_Z + Z * DIM_X * DIM_Y + Y * DIM_X + X + x];
144	}
145
146	template<int X, int Y, int Z>
147	const double& src(const int& x) const
148	{
149	return source[Z * DIM_X * DIM_Y + Y * DIM_X + X + x];
150	}
151
152	private:
153	double *coefficients;
154	double *source;
155	};
156
157	template<typename UPDATER, int DIM>
158	class Benchmark
159	{
160	public:
161	typedef Grid<double, typename Topologies::Cube<DIM>::Topology> GridType;
162
163
164	void run(Coord<DIM> dim, int repeats)
165	{
166	Coord<DIM> coeffDim = dim;
167	coeffDim.c[DIM - 1] *= UPDATER::coefficients();
168	GridType coeff(coeffDim, 0.1);
169	GridType a(dim, 1.0);
170	GridType b(dim, 1.0);
171
172	GridType *oldGrid = &a;
173	GridType *newGrid = &b;
174
175	UPDATER updater;
176
177	long long tStart = getUTtime();
178	std::vector<double*> coefficients(UPDATER::coefficients());
179
180	for (int t = 0; t < repeats; ++t) {
181	for (int z = 1; z < dim.c[2] - 1; ++z) {
182	for (int y = 1; y < dim.c[1] - 1; ++y) {
183	Coord<DIM> c(0, y, z);
184	Coord<DIM> coeffCoord = c;
185
186	updater.step(
187	Neighborhood<4, 4, MY_SIZE>(&coeff.at(coeffCoord), &oldGrid->at(c)),
188	&newGrid->at(c),
189	dim.c[0],
190	dim.c[0] * dim.c[1],
191	1,
192	dim.c[0] - 1);
193
194	// for (int i = 0; i < UPDATER::coefficients(); ++i) {
195	// coefficients[i] = &coeff.at(coeffCoord);
196	// coeffCoord.c[DIM - 1] += dim.c[DIM - 1];
197	// }
198	// updater.step(
199	// &coefficients[0],
200	// &oldGrid->at(c),
201	// &newGrid->at(c),
202	// dim.c[0],
203	// dim.c[0] * dim.c[1],
204	// 1,
205	// dim.c[0] - 1);
206
207	// double *source[9] = {
208	// &oldGrid->at(Coord<DIM>(0, y - 1, z + 1)),
209	// &oldGrid->at(Coord<DIM>(0, y - 1, z )),
210	// &oldGrid->at(Coord<DIM>(0, y - 1, z - 1)),
211	// &oldGrid->at(Coord<DIM>(0, y, z + 1)),
212	// &oldGrid->at(Coord<DIM>(0, y, z )),
213	// &oldGrid->at(Coord<DIM>(0, y, z - 1)),
214	// &oldGrid->at(Coord<DIM>(0, y + 1, z + 1)),
215	// &oldGrid->at(Coord<DIM>(0, y + 1, z )),
216	// &oldGrid->at(Coord<DIM>(0, y + 1, z - 1))
217	// };
218	// updater.update(source, &newGrid->at(c), 1, dim.c[0] - 1);
219	}
220	}
221	std::swap(newGrid, oldGrid);
222	}
223
224	long long tEnd = getUTtime();
225	evaluate(dim, repeats, tEnd - tStart);
226	}
227
228	void exercise()
229	{
230	std::cout << "# " << typeid(UPDATER).name() << "\n";
231	int lastDim = 0;
232	for (int i = 4; i <= 4096; i *= 2) {
233	int intermediateSteps = 8;
234	for (int j = 0; j < intermediateSteps; ++j) {
235	int d = i * std::pow(2, j * (1.0 / intermediateSteps));
236	if (d % 2) {
237	d += 1;
238	}
239
240	if (d > lastDim) {
241	lastDim = d;
242	Coord<DIM> dim;
243	dim.c[0] = d;
244	dim.c[1] = 4;
245	dim.c[2] = 4;
246	// for (int i = 0; i < DIM; ++i)
247	// dim.c[i] = d;
248	int repeats = std::max(1, 10000000 / dim.prod());
249	run(dim, repeats);
250	}
251	}
252	}
253	std::cout << "\n";
254	}
255
256	private:
257	long long getUTtime()
258	{
259	timeval t;
260	gettimeofday(&t, 0);
261	return (long long)t.tv_sec * 1000000 + t.tv_usec;
262	}
263
264	void evaluate(Coord<DIM> dim, int repeats, long long uTime)
265	{
266	double seconds = 1.0 * uTime / 1000 / 1000;
267	Coord<DIM> inner = dim;
268	for (int i = 0; i < DIM; ++i)
269	inner.c[i] -= 2;
270	double gflops = 1.0 * UPDATER().flops() * inner.prod() *
271	repeats / 1000 / 1000 / 1000 / seconds;
272	std::cout << dim.x() << " " << gflops << "\n";
273	}
274
275
276	};
277
278	#define TN 0
279	#define T 1
280	#define TS 2
281	#define N 3
282	#define C 4
283	#define S 5
284	#define BN 6
285	#define B 7
286	#define BS 8
287
288	class Jacobi3D
289	{
290	public:
291	static int coefficients()
292	{
293	return 1;
294	}
295
296	inline void update(double *src, double dst, int startX, int endX)
297	{
298	int x = startX;
299
300	if ((x & 1) == 1) {
301	updateScalar(src, dst, x, x + 1);
302	x += 1;
303	}
304
305	__m128d oneSeventh = _mm_set_pd(1.0/7.0, 1.0/7.0);
306	__m128d buff0 = _mm_loadu_pd(src[C] + x - 1);
307	__m128d same0 = _mm_load_pd(src[C] + x + 0);
308
309	int paddedEndX = endX - 7;
310	for (; x < paddedEndX; x += 8) {
311	// load center row
312	__m128d same1 = _mm_load_pd(src[0] + x + 2);
313	__m128d same2 = _mm_load_pd(src[0] + x + 4);
314	__m128d same3 = _mm_load_pd(src[0] + x + 6);
315	__m128d same4 = _mm_load_pd(src[0] + x + 8);
316
317	// shuffle values obtain left/right neighbors
318	__m128d buff1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
319	__m128d buff2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
320	__m128d buff3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
321	__m128d buff4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
322
323	// load top row
324	__m128d temp0 = _mm_load_pd(src[T] + x + 0);
325	__m128d temp1 = _mm_load_pd(src[T] + x + 2);
326	__m128d temp2 = _mm_load_pd(src[T] + x + 4);
327	__m128d temp3 = _mm_load_pd(src[T] + x + 6);
328
329	// add center row with left...
330	same0 = _mm_add_pd(same0, buff0);
331	same1 = _mm_add_pd(same1, buff1);
332	same2 = _mm_add_pd(same2, buff2);
333	same3 = _mm_add_pd(same3, buff3);
334
335	// ...and right neighbors
336	same0 = _mm_add_pd(same0, buff1);
337	same1 = _mm_add_pd(same1, buff2);
338	same2 = _mm_add_pd(same2, buff3);
339	same3 = _mm_add_pd(same3, buff4);
340
341	// load bottom row
342	buff0 = _mm_load_pd(src[B] + x + 0);
343	buff1 = _mm_load_pd(src[B] + x + 2);
344	buff2 = _mm_load_pd(src[B] + x + 4);
345	buff3 = _mm_load_pd(src[B] + x + 6);
346
347	// add top row
348	same0 = _mm_add_pd(same0, temp0);
349	same1 = _mm_add_pd(same1, temp1);
350	same2 = _mm_add_pd(same2, temp2);
351	same3 = _mm_add_pd(same3, temp3);
352
353	// load north row
354	temp0 = _mm_load_pd(src[N] + x + 0);
355	temp1 = _mm_load_pd(src[N] + x + 2);
356	temp2 = _mm_load_pd(src[N] + x + 4);
357	temp3 = _mm_load_pd(src[N] + x + 6);
358
359	// add bottom row
360	same0 = _mm_add_pd(same0, buff0);
361	same1 = _mm_add_pd(same1, buff1);
362	same2 = _mm_add_pd(same2, buff2);
363	same3 = _mm_add_pd(same3, buff3);
364
365	// load south row
366	buff0 = _mm_load_pd(src[S] + x + 0);
367	buff1 = _mm_load_pd(src[S] + x + 2);
368	buff2 = _mm_load_pd(src[S] + x + 4);
369	buff3 = _mm_load_pd(src[S] + x + 6);
370
371	// add north row
372	same0 = _mm_add_pd(same0, temp0);
373	same1 = _mm_add_pd(same1, temp1);
374	same2 = _mm_add_pd(same2, temp2);
375	same3 = _mm_add_pd(same3, temp3);
376
377	// add south row
378	same0 = _mm_add_pd(same0, buff0);
379	same1 = _mm_add_pd(same1, buff1);
380	same2 = _mm_add_pd(same2, buff2);
381	same3 = _mm_add_pd(same3, buff3);
382
383	// scale down...
384	same0 = _mm_mul_pd(same0, oneSeventh);
385	same1 = _mm_mul_pd(same1, oneSeventh);
386	same2 = _mm_mul_pd(same2, oneSeventh);
387	same3 = _mm_mul_pd(same3, oneSeventh);
388
389	// ...and store
390	_mm_store_pd(dst + x + 0, same0);
391	_mm_store_pd(dst + x + 2, same1);
392	_mm_store_pd(dst + x + 4, same2);
393	_mm_store_pd(dst + x + 6, same3);
394
395	same0 = same4;
396	buff0 = buff4;
397	}
398
399	updateScalar(src, dst, x, endX);
400	}
401
402	inline void updateScalar(double src[9], double dst, int startX, int endX)
403	{
404	for (int x = startX; x < endX; ++x) {
405	dst[x] =
406	(src[S][x] +
407	src[T][x] +
408	src[C][x - 1] +
409	src[C][x] +
410	src[C][x + 1] +
411	src[B][x] +
412	src[N][x]) * (1.0 / 7.0);
413	}
414	}
415
416	int flops()
417	{
418	return 8;
419	}
420	};
421
422
423	class Scalar3D
424	{
425	public:
426	static int coefficients()
427	{
428	return 7;
429	}
430
431	inline void step(double coeff[7], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
432	{
433	for (int x = startX; x < endX; ++x) {
434	dst[x] =
435	coeff[0][x] * src[x - offsetZ] +
436	coeff[1][x] * src[x - offsetY] +
437	coeff[2][x] * src[x - 1] +
438	coeff[3][x] * src[x] +
439	coeff[4][x] * src[x + 1] +
440	coeff[5][x] * src[x + offsetY] +
441	coeff[6][x] * src[x + offsetZ];
442	}
443	}
444
445	int flops()
446	{
447	return 13;
448	}
449	};
450
451	class Vectorized3D
452	{
453	public:
454	static int coefficients()
455	{
456	return 7;
457	}
458
459	inline void step(double *coeff, double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
460	{
461	int x = startX;
462	Scalar3D scalarUpdater;
463
464	if ((x & 1) == 1) {
465	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, x + 1);
466	x += 1;
467	}
468
469	__m128d same0 = _mm_load_pd(src + x + 0);
470	__m128d neig0 = _mm_loadu_pd(src + x + 1);
471
472	int paddedEndX = endX - 7;
473	for (; x < paddedEndX; x += 8) {
474	__m128d same1 = _mm_load_pd(src + x + 2);
475	__m128d same2 = _mm_load_pd(src + x + 4);
476	__m128d same3 = _mm_load_pd(src + x + 6);
477	__m128d same4 = _mm_load_pd(src + x + 8);
478
479	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
480	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
481	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
482	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
483
484	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[3][x + 0]));
485	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[3][x + 2]));
486	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[3][x + 4]));
487	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[3][x + 6]));
488
489	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[2][x + 0]));
490	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[2][x + 2]));
491	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[2][x + 4]));
492	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[2][x + 6]));
493
494	same0 = _mm_add_pd(same0, temp1);
495	same1 = _mm_add_pd(same1, temp2);
496	same2 = _mm_add_pd(same2, temp3);
497	same3 = _mm_add_pd(same3, temp4);
498
499	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[4][x + 0]));
500	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[4][x + 2]));
501	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[4][x + 4]));
502	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[4][x + 6]));
503
504	same0 = _mm_add_pd(same0, temp1);
505	same1 = _mm_add_pd(same1, temp2);
506	same2 = _mm_add_pd(same2, temp3);
507	same3 = _mm_add_pd(same3, temp4);
508
509	neig0 = _mm_load_pd(src + x - offsetZ + 0);
510	neig1 = _mm_load_pd(src + x - offsetZ + 2);
511	neig2 = _mm_load_pd(src + x - offsetZ + 4);
512	neig3 = _mm_load_pd(src + x - offsetZ + 6);
513
514	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][x + 0]));
515	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][x + 2]));
516	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][x + 4]));
517	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][x + 6]));
518
519	same0 = _mm_add_pd(same0, temp1);
520	same1 = _mm_add_pd(same1, temp2);
521	same2 = _mm_add_pd(same2, temp3);
522	same3 = _mm_add_pd(same3, temp4);
523
524	neig0 = _mm_load_pd(src + x - offsetY + 0);
525	neig1 = _mm_load_pd(src + x - offsetY + 2);
526	neig2 = _mm_load_pd(src + x - offsetY + 4);
527	neig3 = _mm_load_pd(src + x - offsetY + 6);
528
529	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[1][x + 0]));
530	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[1][x + 2]));
531	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[1][x + 4]));
532	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[1][x + 6]));
533
534	same0 = _mm_add_pd(same0, temp1);
535	same1 = _mm_add_pd(same1, temp2);
536	same2 = _mm_add_pd(same2, temp3);
537	same3 = _mm_add_pd(same3, temp4);
538
539	neig0 = _mm_load_pd(src + x + offsetY + 0);
540	neig1 = _mm_load_pd(src + x + offsetY + 2);
541	neig2 = _mm_load_pd(src + x + offsetY + 4);
542	neig3 = _mm_load_pd(src + x + offsetY + 6);
543
544	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[5][x + 0]));
545	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[5][x + 2]));
546	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[5][x + 4]));
547	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[5][x + 6]));
548
549	same0 = _mm_add_pd(same0, temp1);
550	same1 = _mm_add_pd(same1, temp2);
551	same2 = _mm_add_pd(same2, temp3);
552	same3 = _mm_add_pd(same3, temp4);
553
554	neig0 = _mm_load_pd(src + x + offsetZ + 0);
555	neig1 = _mm_load_pd(src + x + offsetZ + 2);
556	neig2 = _mm_load_pd(src + x + offsetZ + 4);
557	neig3 = _mm_load_pd(src + x + offsetZ + 6);
558
559	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[6][x + 0]));
560	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[6][x + 2]));
561	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[6][x + 4]));
562	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[6][x + 6]));
563
564	same0 = _mm_add_pd(same0, temp1);
565	same1 = _mm_add_pd(same1, temp2);
566	same2 = _mm_add_pd(same2, temp3);
567	same3 = _mm_add_pd(same3, temp4);
568
569	_mm_store_pd(dst + 0, same0);
570	_mm_store_pd(dst + 2, same1);
571	_mm_store_pd(dst + 4, same2);
572	_mm_store_pd(dst + 6, same3);
573
574	same0 = same4;
575	neig0 = neig4;
576
577	// dst[x] =
578	// coeff[0][x] * src[x - offsetZ] +
579	// coeff[1][x] * src[x - offsetY] +
580	// coeff[2][x] * src[x - 1] +
581	// coeff[3][x] * src[x] +
582	// coeff[4][x] * src[x + 1] +
583	// coeff[5][x] * src[x + offsetY] +
584	// coeff[6][x] * src[x + offsetZ];
585	}
586
587	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
588	}
589
590	int flops()
591	{
592	return 13;
593	}
594	};
595
596	class ExtendedScalar3D
597	{
598	public:
599	static int coefficients()
600	{
601	return 13;
602	}
603
604	inline void step(double coeff[13], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
605	{
606	for (int x = startX; x < endX; ++x) {
607	dst[x] =
608	coeff[ 0][x] * src[x - offsetZ - offsetY] +
609	coeff[ 1][x] * src[x - offsetZ] +
610	coeff[ 2][x] * src[x - offsetZ + offsetY] +
611	coeff[ 3][x] * src[x - offsetY] +
612	coeff[ 4][x] * src[x - 1] +
613	coeff[ 5][x] * src[x] +
614	coeff[ 6][x] * src[x + 1] +
615	coeff[ 7][x] * src[x + offsetY] +
616	coeff[ 8][x] * src[x + offsetZ - offsetY] +
617	coeff[ 9][x] * src[x + offsetZ] +
618	coeff[10][x] * src[x + offsetZ + offsetY] +
619	coeff[11][x - offsetZ - offsetY] +
620	coeff[11][x - offsetZ] +
621	coeff[11][x - offsetZ + offsetY] +
622	coeff[11][x - offsetY] +
623	coeff[11][x] +
624	coeff[11][x + offsetY] +
625	coeff[11][x + offsetZ - offsetY] +
626	coeff[11][x + offsetZ] +
627	coeff[11][x + offsetZ + offsetY] +
628	coeff[12][x - offsetZ - offsetY] +
629	coeff[12][x - offsetZ] +
630	coeff[12][x - offsetZ + offsetY] +
631	coeff[12][x - offsetY] +
632	coeff[12][x] +
633	coeff[12][x + offsetY] +
634	coeff[12][x + offsetZ - offsetY] +
635	coeff[12][x + offsetZ] +
636	coeff[12][x + offsetZ + offsetY];
637	}
638	}
639
640	int flops()
641	{
642	return 40;
643	}
644	};
645
646	class ExtendedVectorized3D
647	{
648	public:
649	static int coefficients()
650	{
651	return 13;
652	}
653
654	inline void step(double coeff[13], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
655	{
656	int x = startX;
657	ExtendedScalar3D scalarUpdater;
658
659	if ((x & 1) == 1) {
660	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, x + 1);
661	x += 1;
662	}
663
664	__m128d same0 = _mm_load_pd(src + x + 0);
665	__m128d neig0 = _mm_loadu_pd(src + x + 1);
666
667	int paddedEndX = endX - 7;
668	for (; x < paddedEndX; x += 8) {
669	__m128d same1 = _mm_load_pd(src + x + 2);
670	__m128d same2 = _mm_load_pd(src + x + 4);
671	__m128d same3 = _mm_load_pd(src + x + 6);
672	__m128d same4 = _mm_load_pd(src + x + 8);
673
674	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
675	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
676	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
677	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
678
679	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[3][x + 0]));
680	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[3][x + 2]));
681	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[3][x + 4]));
682	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[3][x + 6]));
683
684	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[2][x + 0]));
685	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[2][x + 2]));
686	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[2][x + 4]));
687	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[2][x + 6]));
688
689	same0 = _mm_add_pd(same0, temp1);
690	same1 = _mm_add_pd(same1, temp2);
691	same2 = _mm_add_pd(same2, temp3);
692	same3 = _mm_add_pd(same3, temp4);
693
694	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[4][x + 0]));
695	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[4][x + 2]));
696	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[4][x + 4]));
697	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[4][x + 6]));
698
699	same0 = _mm_add_pd(same0, temp1);
700	same1 = _mm_add_pd(same1, temp2);
701	same2 = _mm_add_pd(same2, temp3);
702	same3 = _mm_add_pd(same3, temp4);
703
704	neig0 = _mm_load_pd(src + x - offsetZ + 0);
705	neig1 = _mm_load_pd(src + x - offsetZ + 2);
706	neig2 = _mm_load_pd(src + x - offsetZ + 4);
707	neig3 = _mm_load_pd(src + x - offsetZ + 6);
708
709	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][x + 0]));
710	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][x + 2]));
711	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][x + 4]));
712	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][x + 6]));
713
714	same0 = _mm_add_pd(same0, temp1);
715	same1 = _mm_add_pd(same1, temp2);
716	same2 = _mm_add_pd(same2, temp3);
717	same3 = _mm_add_pd(same3, temp4);
718
719	neig0 = _mm_load_pd(src + x - offsetY + 0);
720	neig1 = _mm_load_pd(src + x - offsetY + 2);
721	neig2 = _mm_load_pd(src + x - offsetY + 4);
722	neig3 = _mm_load_pd(src + x - offsetY + 6);
723
724	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[1][x + 0]));
725	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[1][x + 2]));
726	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[1][x + 4]));
727	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[1][x + 6]));
728
729	same0 = _mm_add_pd(same0, temp1);
730	same1 = _mm_add_pd(same1, temp2);
731	same2 = _mm_add_pd(same2, temp3);
732	same3 = _mm_add_pd(same3, temp4);
733
734	neig0 = _mm_load_pd(src + x + offsetY + 0);
735	neig1 = _mm_load_pd(src + x + offsetY + 2);
736	neig2 = _mm_load_pd(src + x + offsetY + 4);
737	neig3 = _mm_load_pd(src + x + offsetY + 6);
738
739	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[5][x + 0]));
740	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[5][x + 2]));
741	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[5][x + 4]));
742	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[5][x + 6]));
743
744	same0 = _mm_add_pd(same0, temp1);
745	same1 = _mm_add_pd(same1, temp2);
746	same2 = _mm_add_pd(same2, temp3);
747	same3 = _mm_add_pd(same3, temp4);
748
749	//xxxxxxxxxxxxx
750	neig0 = _mm_load_pd(src + x + offsetZ + 0);
751	neig1 = _mm_load_pd(src + x + offsetZ + 2);
752	neig2 = _mm_load_pd(src + x + offsetZ + 4);
753	neig3 = _mm_load_pd(src + x + offsetZ + 6);
754
755	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[6][x + 0]));
756	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[6][x + 2]));
757	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[6][x + 4]));
758	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[6][x + 6]));
759
760	same0 = _mm_add_pd(same0, temp1);
761	same1 = _mm_add_pd(same1, temp2);
762	same2 = _mm_add_pd(same2, temp3);
763	same3 = _mm_add_pd(same3, temp4);
764
765	//xxxxxxxxxxxxx
766	neig0 = _mm_load_pd(src + x - offsetZ - offsetY + 0);
767	neig1 = _mm_load_pd(src + x - offsetZ - offsetY + 2);
768	neig2 = _mm_load_pd(src + x - offsetZ - offsetY + 4);
769	neig3 = _mm_load_pd(src + x - offsetZ - offsetY + 6);
770
771	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[7][x + 0]));
772	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[7][x + 2]));
773	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[7][x + 4]));
774	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[7][x + 6]));
775
776	same0 = _mm_add_pd(same0, temp1);
777	same1 = _mm_add_pd(same1, temp2);
778	same2 = _mm_add_pd(same2, temp3);
779	same3 = _mm_add_pd(same3, temp4);
780
781	//xxxxxxxxxxxxx
782	neig0 = _mm_load_pd(src + x - offsetZ + offsetY + 0);
783	neig1 = _mm_load_pd(src + x - offsetZ + offsetY + 2);
784	neig2 = _mm_load_pd(src + x - offsetZ + offsetY + 4);
785	neig3 = _mm_load_pd(src + x - offsetZ + offsetY + 6);
786
787	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[8][x + 0]));
788	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[8][x + 2]));
789	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[8][x + 4]));
790	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[8][x + 6]));
791
792	same0 = _mm_add_pd(same0, temp1);
793	same1 = _mm_add_pd(same1, temp2);
794	same2 = _mm_add_pd(same2, temp3);
795	same3 = _mm_add_pd(same3, temp4);
796
797	//xxxxxxxxxxxxx
798	neig0 = _mm_load_pd(src + x + offsetZ - offsetY + 0);
799	neig1 = _mm_load_pd(src + x + offsetZ - offsetY + 2);
800	neig2 = _mm_load_pd(src + x + offsetZ - offsetY + 4);
801	neig3 = _mm_load_pd(src + x + offsetZ - offsetY + 6);
802
803	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[9][x + 0]));
804	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[9][x + 2]));
805	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[9][x + 4]));
806	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[9][x + 6]));
807
808	same0 = _mm_add_pd(same0, temp1);
809	same1 = _mm_add_pd(same1, temp2);
810	same2 = _mm_add_pd(same2, temp3);
811	same3 = _mm_add_pd(same3, temp4);
812
813	//xxxxxxxxxxxxx
814	neig0 = _mm_load_pd(src + x + offsetZ + offsetY + 0);
815	neig1 = _mm_load_pd(src + x + offsetZ + offsetY + 2);
816	neig2 = _mm_load_pd(src + x + offsetZ + offsetY + 4);
817	neig3 = _mm_load_pd(src + x + offsetZ + offsetY + 6);
818
819	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[10][x + 0]));
820	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[10][x + 2]));
821	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[10][x + 4]));
822	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[10][x + 6]));
823
824	same0 = _mm_add_pd(same0, temp1);
825	same1 = _mm_add_pd(same1, temp2);
826	same2 = _mm_add_pd(same2, temp3);
827	same3 = _mm_add_pd(same3, temp4);
828
829	//xxxxxxxxxxxxx
830	neig0 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 0);
831	neig1 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 2);
832	neig2 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 4);
833	neig3 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 6);
834
835	same0 = _mm_add_pd(same0, neig0);
836	same1 = _mm_add_pd(same1, neig1);
837	same2 = _mm_add_pd(same2, neig2);
838	same3 = _mm_add_pd(same3, neig3);
839
840	//xxxxxxxxxxxxx
841	neig0 = _mm_load_pd(coeff[11] + x - offsetZ + 0);
842	neig1 = _mm_load_pd(coeff[11] + x - offsetZ + 2);
843	neig2 = _mm_load_pd(coeff[11] + x - offsetZ + 4);
844	neig3 = _mm_load_pd(coeff[11] + x - offsetZ + 6);
845
846	same0 = _mm_add_pd(same0, neig0);
847	same1 = _mm_add_pd(same1, neig1);
848	same2 = _mm_add_pd(same2, neig2);
849	same3 = _mm_add_pd(same3, neig3);
850
851	//xxxxxxxxxxxxx
852	neig0 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 0);
853	neig1 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 2);
854	neig2 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 4);
855	neig3 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 6);
856
857	same0 = _mm_add_pd(same0, neig0);
858	same1 = _mm_add_pd(same1, neig1);
859	same2 = _mm_add_pd(same2, neig2);
860	same3 = _mm_add_pd(same3, neig3);
861
862	//xxxxxxxxxxxxx
863	neig0 = _mm_load_pd(coeff[11] + x - offsetY + 0);
864	neig1 = _mm_load_pd(coeff[11] + x - offsetY + 2);
865	neig2 = _mm_load_pd(coeff[11] + x - offsetY + 4);
866	neig3 = _mm_load_pd(coeff[11] + x - offsetY + 6);
867
868	same0 = _mm_add_pd(same0, neig0);
869	same1 = _mm_add_pd(same1, neig1);
870	same2 = _mm_add_pd(same2, neig2);
871	same3 = _mm_add_pd(same3, neig3);
872
873	//xxxxxxxxxxxxx
874	neig0 = _mm_load_pd(coeff[11] + offsetY + x + 0);
875	neig1 = _mm_load_pd(coeff[11] + offsetY + x + 2);
876	neig2 = _mm_load_pd(coeff[11] + offsetY + x + 4);
877	neig3 = _mm_load_pd(coeff[11] + offsetY + x + 6);
878
879	same0 = _mm_add_pd(same0, neig0);
880	same1 = _mm_add_pd(same1, neig1);
881	same2 = _mm_add_pd(same2, neig2);
882	same3 = _mm_add_pd(same3, neig3);
883
884	//xxxxxxxxxxxxx
885	neig0 = _mm_load_pd(coeff[11] + x + 0);
886	neig1 = _mm_load_pd(coeff[11] + x + 2);
887	neig2 = _mm_load_pd(coeff[11] + x + 4);
888	neig3 = _mm_load_pd(coeff[11] + x + 6);
889
890	same0 = _mm_add_pd(same0, neig0);
891	same1 = _mm_add_pd(same1, neig1);
892	same2 = _mm_add_pd(same2, neig2);
893	same3 = _mm_add_pd(same3, neig3);
894
895	//xxxxxxxxxxxxx
896	neig0 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 0);
897	neig1 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 2);
898	neig2 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 4);
899	neig3 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 6);
900
901	same0 = _mm_add_pd(same0, neig0);
902	same1 = _mm_add_pd(same1, neig1);
903	same2 = _mm_add_pd(same2, neig2);
904	same3 = _mm_add_pd(same3, neig3);
905
906	//xxxxxxxxxxxxx
907	neig0 = _mm_load_pd(coeff[11] + x + offsetZ + 0);
908	neig1 = _mm_load_pd(coeff[11] + x + offsetZ + 2);
909	neig2 = _mm_load_pd(coeff[11] + x + offsetZ + 4);
910	neig3 = _mm_load_pd(coeff[11] + x + offsetZ + 6);
911
912	same0 = _mm_add_pd(same0, neig0);
913	same1 = _mm_add_pd(same1, neig1);
914	same2 = _mm_add_pd(same2, neig2);
915	same3 = _mm_add_pd(same3, neig3);
916
917	//xxxxxxxxxxxxx
918	neig0 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 0);
919	neig1 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 2);
920	neig2 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 4);
921	neig3 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 6);
922
923	same0 = _mm_add_pd(same0, neig0);
924	same1 = _mm_add_pd(same1, neig1);
925	same2 = _mm_add_pd(same2, neig2);
926	same3 = _mm_add_pd(same3, neig3);
927
928	//xxxxxxxxxxxxx
929	neig0 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 0);
930	neig1 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 2);
931	neig2 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 4);
932	neig3 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 6);
933
934	same0 = _mm_add_pd(same0, neig0);
935	same1 = _mm_add_pd(same1, neig1);
936	same2 = _mm_add_pd(same2, neig2);
937	same3 = _mm_add_pd(same3, neig3);
938
939	//xxxxxxxxxxxxx
940	neig0 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 0);
941	neig1 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 2);
942	neig2 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 4);
943	neig3 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 6);
944
945	same0 = _mm_add_pd(same0, neig0);
946	same1 = _mm_add_pd(same1, neig1);
947	same2 = _mm_add_pd(same2, neig2);
948	same3 = _mm_add_pd(same3, neig3);
949
950	//xxxxxxxxxxxxx
951	neig0 = _mm_load_pd(coeff[12] + x - offsetZ + 0);
952	neig1 = _mm_load_pd(coeff[12] + x - offsetZ + 2);
953	neig2 = _mm_load_pd(coeff[12] + x - offsetZ + 4);
954	neig3 = _mm_load_pd(coeff[12] + x - offsetZ + 6);
955
956	same0 = _mm_add_pd(same0, neig0);
957	same1 = _mm_add_pd(same1, neig1);
958	same2 = _mm_add_pd(same2, neig2);
959	same3 = _mm_add_pd(same3, neig3);
960
961	//xxxxxxxxxxxxx
962	neig0 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 0);
963	neig1 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 2);
964	neig2 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 4);
965	neig3 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 6);
966
967	same0 = _mm_add_pd(same0, neig0);
968	same1 = _mm_add_pd(same1, neig1);
969	same2 = _mm_add_pd(same2, neig2);
970	same3 = _mm_add_pd(same3, neig3);
971
972	//xxxxxxxxxxxxx
973	neig0 = _mm_load_pd(coeff[12] + x - offsetY + 0);
974	neig1 = _mm_load_pd(coeff[12] + x - offsetY + 2);
975	neig2 = _mm_load_pd(coeff[12] + x - offsetY + 4);
976	neig3 = _mm_load_pd(coeff[12] + x - offsetY + 6);
977
978	same0 = _mm_add_pd(same0, neig0);
979	same1 = _mm_add_pd(same1, neig1);
980	same2 = _mm_add_pd(same2, neig2);
981	same3 = _mm_add_pd(same3, neig3);
982
983	//xxxxxxxxxxxxx
984	neig0 = _mm_load_pd(coeff[12] + offsetY + x + 0);
985	neig1 = _mm_load_pd(coeff[12] + offsetY + x + 2);
986	neig2 = _mm_load_pd(coeff[12] + offsetY + x + 4);
987	neig3 = _mm_load_pd(coeff[12] + offsetY + x + 6);
988
989	same0 = _mm_add_pd(same0, neig0);
990	same1 = _mm_add_pd(same1, neig1);
991	same2 = _mm_add_pd(same2, neig2);
992	same3 = _mm_add_pd(same3, neig3);
993
994	//xxxxxxxxxxxxx
995	neig0 = _mm_load_pd(coeff[12] + x + 0);
996	neig1 = _mm_load_pd(coeff[12] + x + 2);
997	neig2 = _mm_load_pd(coeff[12] + x + 4);
998	neig3 = _mm_load_pd(coeff[12] + x + 6);
999
1000	same0 = _mm_add_pd(same0, neig0);
1001	same1 = _mm_add_pd(same1, neig1);
1002	same2 = _mm_add_pd(same2, neig2);
1003	same3 = _mm_add_pd(same3, neig3);
1004
1005	//xxxxxxxxxxxxx
1006	neig0 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 0);
1007	neig1 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 2);
1008	neig2 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 4);
1009	neig3 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 6);
1010
1011	same0 = _mm_add_pd(same0, neig0);
1012	same1 = _mm_add_pd(same1, neig1);
1013	same2 = _mm_add_pd(same2, neig2);
1014	same3 = _mm_add_pd(same3, neig3);
1015
1016	//xxxxxxxxxxxxx
1017	neig0 = _mm_load_pd(coeff[12] + x + offsetZ + 0);
1018	neig1 = _mm_load_pd(coeff[12] + x + offsetZ + 2);
1019	neig2 = _mm_load_pd(coeff[12] + x + offsetZ + 4);
1020	neig3 = _mm_load_pd(coeff[12] + x + offsetZ + 6);
1021
1022	same0 = _mm_add_pd(same0, neig0);
1023	same1 = _mm_add_pd(same1, neig1);
1024	same2 = _mm_add_pd(same2, neig2);
1025	same3 = _mm_add_pd(same3, neig3);
1026
1027	//xxxxxxxxxxxxx
1028	neig0 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 0);
1029	neig1 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 2);
1030	neig2 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 4);
1031	neig3 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 6);
1032
1033	same0 = _mm_add_pd(same0, neig0);
1034	same1 = _mm_add_pd(same1, neig1);
1035	same2 = _mm_add_pd(same2, neig2);
1036	same3 = _mm_add_pd(same3, neig3);
1037
1038	//yyyyyyyyyyyyy
1039	_mm_store_pd(dst + x + 0, same0);
1040	_mm_store_pd(dst + x + 2, same1);
1041	_mm_store_pd(dst + x + 4, same2);
1042	_mm_store_pd(dst + x + 6, same3);
1043
1044	same0 = same4;
1045	neig0 = neig4;
1046
1047	// dst[x] =
1048	// coeff[0][x] * src[x - offsetZ] +
1049	// coeff[1][x] * src[x - offsetY] +
1050	// coeff[2][x] * src[x - 1] +
1051	// coeff[3][x] * src[x] +
1052	// coeff[4][x] * src[x + 1] +
1053	// coeff[5][x] * src[x + offsetY] +
1054	// coeff[6][x] * src[x + offsetZ];
1055	}
1056
1057	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
1058	}
1059
1060	int flops()
1061	{
1062	return 40;
1063	}
1064	};
1065
1066	template<int DIM_X, int DIM_Y, int DIM_Z>
1067	class ExtendedVectorized3DFixed
1068	{
1069	public:
1070	static int coefficients()
1071	{
1072	return 13;
1073	}
1074
1075	inline void step(double coeff[13], double src, double *dst, int unusedOffsetY, int unusedOffsetZ, int startX, int endX)
1076	{
1077	const int SLICE_SIZE = DIM_X * DIM_Y;
1078	const int TOTAL_SIZE = DIM_X * DIM_Y * DIM_Z;
1079	const int offsetY = DIM_X;
1080	const int offsetZ = SLICE_SIZE;
1081
1082	int x = startX;
1083	ExtendedScalar3D scalarUpdater;
1084
1085	if ((x & 1) == 1) {
1086	scalarUpdater.step(coeff, src, dst, DIM_Y, offsetZ, x, x + 1);
1087	x += 1;
1088	}
1089
1090	__m128d same0 = _mm_load_pd(src + x + 0);
1091	__m128d neig0 = _mm_loadu_pd(src + x + 1);
1092
1093	int paddedEndX = endX - 7;
1094	for (; x < paddedEndX; x += 8) {
1095	__m128d same1 = _mm_load_pd(src + x + 2);
1096	__m128d same2 = _mm_load_pd(src + x + 4);
1097	__m128d same3 = _mm_load_pd(src + x + 6);
1098	__m128d same4 = _mm_load_pd(src + x + 8);
1099
1100	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
1101	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
1102	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
1103	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
1104
1105	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 0]));
1106	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 2]));
1107	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 4]));
1108	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 6]));
1109
1110	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 0]));
1111	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 2]));
1112	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 4]));
1113	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 6]));
1114
1115	same0 = _mm_add_pd(same0, temp1);
1116	same1 = _mm_add_pd(same1, temp2);
1117	same2 = _mm_add_pd(same2, temp3);
1118	same3 = _mm_add_pd(same3, temp4);
1119
1120	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 0]));
1121	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 2]));
1122	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 4]));
1123	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 6]));
1124
1125	same0 = _mm_add_pd(same0, temp1);
1126	same1 = _mm_add_pd(same1, temp2);
1127	same2 = _mm_add_pd(same2, temp3);
1128	same3 = _mm_add_pd(same3, temp4);
1129
1130	neig0 = _mm_load_pd(src + x - offsetZ + 0);
1131	neig1 = _mm_load_pd(src + x - offsetZ + 2);
1132	neig2 = _mm_load_pd(src + x - offsetZ + 4);
1133	neig3 = _mm_load_pd(src + x - offsetZ + 6);
1134
1135	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 0]));
1136	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 2]));
1137	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 4]));
1138	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 6]));
1139
1140	same0 = _mm_add_pd(same0, temp1);
1141	same1 = _mm_add_pd(same1, temp2);
1142	same2 = _mm_add_pd(same2, temp3);
1143	same3 = _mm_add_pd(same3, temp4);
1144
1145	neig0 = _mm_load_pd(src + x - offsetY + 0);
1146	neig1 = _mm_load_pd(src + x - offsetY + 2);
1147	neig2 = _mm_load_pd(src + x - offsetY + 4);
1148	neig3 = _mm_load_pd(src + x - offsetY + 6);
1149
1150	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 0]));
1151	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 2]));
1152	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 4]));
1153	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 6]));
1154
1155	same0 = _mm_add_pd(same0, temp1);
1156	same1 = _mm_add_pd(same1, temp2);
1157	same2 = _mm_add_pd(same2, temp3);
1158	same3 = _mm_add_pd(same3, temp4);
1159
1160	neig0 = _mm_load_pd(src + x + offsetY + 0);
1161	neig1 = _mm_load_pd(src + x + offsetY + 2);
1162	neig2 = _mm_load_pd(src + x + offsetY + 4);
1163	neig3 = _mm_load_pd(src + x + offsetY + 6);
1164
1165	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 0]));
1166	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 2]));
1167	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 4]));
1168	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 6]));
1169
1170	same0 = _mm_add_pd(same0, temp1);
1171	same1 = _mm_add_pd(same1, temp2);
1172	same2 = _mm_add_pd(same2, temp3);
1173	same3 = _mm_add_pd(same3, temp4);
1174
1175	//xxxxxxxxxxxxx
1176	neig0 = _mm_load_pd(src + x + offsetZ + 0);
1177	neig1 = _mm_load_pd(src + x + offsetZ + 2);
1178	neig2 = _mm_load_pd(src + x + offsetZ + 4);
1179	neig3 = _mm_load_pd(src + x + offsetZ + 6);
1180
1181	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 0]));
1182	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 2]));
1183	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 4]));
1184	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 6]));
1185
1186	same0 = _mm_add_pd(same0, temp1);
1187	same1 = _mm_add_pd(same1, temp2);
1188	same2 = _mm_add_pd(same2, temp3);
1189	same3 = _mm_add_pd(same3, temp4);
1190
1191	//xxxxxxxxxxxxx
1192	neig0 = _mm_load_pd(src + x - offsetZ - offsetY + 0);
1193	neig1 = _mm_load_pd(src + x - offsetZ - offsetY + 2);
1194	neig2 = _mm_load_pd(src + x - offsetZ - offsetY + 4);
1195	neig3 = _mm_load_pd(src + x - offsetZ - offsetY + 6);
1196
1197	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 0]));
1198	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 2]));
1199	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 4]));
1200	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 6]));
1201
1202	same0 = _mm_add_pd(same0, temp1);
1203	same1 = _mm_add_pd(same1, temp2);
1204	same2 = _mm_add_pd(same2, temp3);
1205	same3 = _mm_add_pd(same3, temp4);
1206
1207	//xxxxxxxxxxxxx
1208	neig0 = _mm_load_pd(src + x - offsetZ + offsetY + 0);
1209	neig1 = _mm_load_pd(src + x - offsetZ + offsetY + 2);
1210	neig2 = _mm_load_pd(src + x - offsetZ + offsetY + 4);
1211	neig3 = _mm_load_pd(src + x - offsetZ + offsetY + 6);
1212
1213	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 0]));
1214	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 2]));
1215	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 4]));
1216	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 6]));
1217
1218	same0 = _mm_add_pd(same0, temp1);
1219	same1 = _mm_add_pd(same1, temp2);
1220	same2 = _mm_add_pd(same2, temp3);
1221	same3 = _mm_add_pd(same3, temp4);
1222
1223	//xxxxxxxxxxxxx
1224	neig0 = _mm_load_pd(src + x + offsetZ - offsetY + 0);
1225	neig1 = _mm_load_pd(src + x + offsetZ - offsetY + 2);
1226	neig2 = _mm_load_pd(src + x + offsetZ - offsetY + 4);
1227	neig3 = _mm_load_pd(src + x + offsetZ - offsetY + 6);
1228
1229	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 0]));
1230	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 2]));
1231	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 4]));
1232	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 6]));
1233
1234	same0 = _mm_add_pd(same0, temp1);
1235	same1 = _mm_add_pd(same1, temp2);
1236	same2 = _mm_add_pd(same2, temp3);
1237	same3 = _mm_add_pd(same3, temp4);
1238
1239	//xxxxxxxxxxxxx
1240	neig0 = _mm_load_pd(src + x + offsetZ + offsetY + 0);
1241	neig1 = _mm_load_pd(src + x + offsetZ + offsetY + 2);
1242	neig2 = _mm_load_pd(src + x + offsetZ + offsetY + 4);
1243	neig3 = _mm_load_pd(src + x + offsetZ + offsetY + 6);
1244
1245	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 0]));
1246	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 2]));
1247	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 4]));
1248	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 6]));
1249
1250	same0 = _mm_add_pd(same0, temp1);
1251	same1 = _mm_add_pd(same1, temp2);
1252	same2 = _mm_add_pd(same2, temp3);
1253	same3 = _mm_add_pd(same3, temp4);
1254
1255	//xxxxxxxxxxxxx
1256	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 0);
1257	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 2);
1258	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 4);
1259	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 6);
1260
1261	same0 = _mm_add_pd(same0, neig0);
1262	same1 = _mm_add_pd(same1, neig1);
1263	same2 = _mm_add_pd(same2, neig2);
1264	same3 = _mm_add_pd(same3, neig3);
1265
1266	//xxxxxxxxxxxxx
1267	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 0);
1268	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 2);
1269	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 4);
1270	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 6);
1271
1272	same0 = _mm_add_pd(same0, neig0);
1273	same1 = _mm_add_pd(same1, neig1);
1274	same2 = _mm_add_pd(same2, neig2);
1275	same3 = _mm_add_pd(same3, neig3);
1276
1277	//xxxxxxxxxxxxx
1278	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 0);
1279	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 2);
1280	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 4);
1281	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 6);
1282
1283	same0 = _mm_add_pd(same0, neig0);
1284	same1 = _mm_add_pd(same1, neig1);
1285	same2 = _mm_add_pd(same2, neig2);
1286	same3 = _mm_add_pd(same3, neig3);
1287
1288	//xxxxxxxxxxxxx
1289	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 0);
1290	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 2);
1291	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 4);
1292	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 6);
1293
1294	same0 = _mm_add_pd(same0, neig0);
1295	same1 = _mm_add_pd(same1, neig1);
1296	same2 = _mm_add_pd(same2, neig2);
1297	same3 = _mm_add_pd(same3, neig3);
1298
1299	//xxxxxxxxxxxxx
1300	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 0);
1301	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 2);
1302	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 4);
1303	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 6);
1304
1305	same0 = _mm_add_pd(same0, neig0);
1306	same1 = _mm_add_pd(same1, neig1);
1307	same2 = _mm_add_pd(same2, neig2);
1308	same3 = _mm_add_pd(same3, neig3);
1309
1310	//xxxxxxxxxxxxx
1311	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 0);
1312	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 2);
1313	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 4);
1314	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 6);
1315
1316	same0 = _mm_add_pd(same0, neig0);
1317	same1 = _mm_add_pd(same1, neig1);
1318	same2 = _mm_add_pd(same2, neig2);
1319	same3 = _mm_add_pd(same3, neig3);
1320
1321	//xxxxxxxxxxxxx
1322	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 0);
1323	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 2);
1324	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 4);
1325	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 6);
1326
1327	same0 = _mm_add_pd(same0, neig0);
1328	same1 = _mm_add_pd(same1, neig1);
1329	same2 = _mm_add_pd(same2, neig2);
1330	same3 = _mm_add_pd(same3, neig3);
1331
1332	//xxxxxxxxxxxxx
1333	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 0);
1334	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 2);
1335	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 4);
1336	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 6);
1337
1338	same0 = _mm_add_pd(same0, neig0);
1339	same1 = _mm_add_pd(same1, neig1);
1340	same2 = _mm_add_pd(same2, neig2);
1341	same3 = _mm_add_pd(same3, neig3);
1342
1343	//xxxxxxxxxxxxx
1344	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 0);
1345	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 2);
1346	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 4);
1347	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 6);
1348
1349	same0 = _mm_add_pd(same0, neig0);
1350	same1 = _mm_add_pd(same1, neig1);
1351	same2 = _mm_add_pd(same2, neig2);
1352	same3 = _mm_add_pd(same3, neig3);
1353
1354	//xxxxxxxxxxxxx
1355	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 0);
1356	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 2);
1357	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 4);
1358	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 6);
1359
1360	same0 = _mm_add_pd(same0, neig0);
1361	same1 = _mm_add_pd(same1, neig1);
1362	same2 = _mm_add_pd(same2, neig2);
1363	same3 = _mm_add_pd(same3, neig3);
1364
1365	//xxxxxxxxxxxxx
1366	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 0);
1367	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 2);
1368	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 4);
1369	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 6);
1370
1371	same0 = _mm_add_pd(same0, neig0);
1372	same1 = _mm_add_pd(same1, neig1);
1373	same2 = _mm_add_pd(same2, neig2);
1374	same3 = _mm_add_pd(same3, neig3);
1375
1376	//xxxxxxxxxxxxx
1377	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 0);
1378	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 2);
1379	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 4);
1380	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 6);
1381
1382	same0 = _mm_add_pd(same0, neig0);
1383	same1 = _mm_add_pd(same1, neig1);
1384	same2 = _mm_add_pd(same2, neig2);
1385	same3 = _mm_add_pd(same3, neig3);
1386
1387	//xxxxxxxxxxxxx
1388	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 0);
1389	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 2);
1390	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 4);
1391	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 6);
1392
1393	same0 = _mm_add_pd(same0, neig0);
1394	same1 = _mm_add_pd(same1, neig1);
1395	same2 = _mm_add_pd(same2, neig2);
1396	same3 = _mm_add_pd(same3, neig3);
1397
1398	//xxxxxxxxxxxxx
1399	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 0);
1400	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 2);
1401	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 4);
1402	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 6);
1403
1404	same0 = _mm_add_pd(same0, neig0);
1405	same1 = _mm_add_pd(same1, neig1);
1406	same2 = _mm_add_pd(same2, neig2);
1407	same3 = _mm_add_pd(same3, neig3);
1408
1409	//xxxxxxxxxxxxx
1410	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 0);
1411	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 2);
1412	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 4);
1413	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 6);
1414
1415	same0 = _mm_add_pd(same0, neig0);
1416	same1 = _mm_add_pd(same1, neig1);
1417	same2 = _mm_add_pd(same2, neig2);
1418	same3 = _mm_add_pd(same3, neig3);
1419
1420	//xxxxxxxxxxxxx
1421	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 0);
1422	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 2);
1423	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 4);
1424	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 6);
1425
1426	same0 = _mm_add_pd(same0, neig0);
1427	same1 = _mm_add_pd(same1, neig1);
1428	same2 = _mm_add_pd(same2, neig2);
1429	same3 = _mm_add_pd(same3, neig3);
1430
1431	//xxxxxxxxxxxxx
1432	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 0);
1433	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 2);
1434	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 4);
1435	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 6);
1436
1437	same0 = _mm_add_pd(same0, neig0);
1438	same1 = _mm_add_pd(same1, neig1);
1439	same2 = _mm_add_pd(same2, neig2);
1440	same3 = _mm_add_pd(same3, neig3);
1441
1442	//xxxxxxxxxxxxx
1443	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 0);
1444	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 2);
1445	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 4);
1446	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 6);
1447
1448	same0 = _mm_add_pd(same0, neig0);
1449	same1 = _mm_add_pd(same1, neig1);
1450	same2 = _mm_add_pd(same2, neig2);
1451	same3 = _mm_add_pd(same3, neig3);
1452
1453	//yyyyyyyyyyyyy
1454	_mm_store_pd(dst + x + 0, same0);
1455	_mm_store_pd(dst + x + 2, same1);
1456	_mm_store_pd(dst + x + 4, same2);
1457	_mm_store_pd(dst + x + 6, same3);
1458
1459	same0 = same4;
1460	neig0 = neig4;
1461
1462	// dst[x] =
1463	// coeff[0][x] * src[x - offsetZ] +
1464	// coeff[1][x] * src[x - offsetY] +
1465	// coeff[2][x] * src[x - 1] +
1466	// coeff[3][x] * src[x] +
1467	// coeff[4][x] * src[x + 1] +
1468	// coeff[5][x] * src[x + offsetY] +
1469	// coeff[6][x] * src[x + offsetZ];
1470	}
1471
1472	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
1473	}
1474
1475	int flops()
1476	{
1477	return 40;
1478	}
1479	};
1480
1481	template<int DIM_X, int DIM_Y, int DIM_Z>
1482	class ExtendedVectorized3DNextGen
1483	{
1484	public:
1485	static int coefficients()
1486	{
1487	return 13;
1488	}
1489
1490	template<class NEIGHBORHOOD>
1491	inline void step(const NEIGHBORHOOD& hood, double *dst, int unusedOffsetY, int unusedOffsetZ, int startX, int endX)
1492	{
1493	const int SLICE_SIZE = DIM_X * DIM_Y;
1494	const int TOTAL_SIZE = DIM_X * DIM_Y * DIM_Z;
1495	const int offsetY = DIM_X;
1496	const int offsetZ = SLICE_SIZE;
1497
1498	int x = startX;
1499
1500	if ((x & 1) == 1) {
1501	stepScalar(hood, dst, DIM_Y, offsetZ, x, x + 1);
1502	x += 1;
1503	}
1504
1505	__m128d same0 = _mm_load_pd(&(hood.template src<0, 0, 0>(x)));
1506	__m128d neig0 = _mm_loadu_pd(&hood.template src<1, 0, 0>(x));
1507
1508	int paddedEndX = endX - 7;
1509	for (; x < paddedEndX; x += 8) {
1510	__m128d same1 = _mm_load_pd(&hood.template src<2, 0, 0>(x));
1511	__m128d same2 = _mm_load_pd(&hood.template src<4, 0, 0>(x));
1512	__m128d same3 = _mm_load_pd(&hood.template src<6, 0, 0>(x));
1513	__m128d same4 = _mm_load_pd(&hood.template src<8, 0, 0>(x));
1514
1515	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
1516	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
1517	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
1518	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
1519
1520	same0 = _mm_mul_pd(same0, _mm_load_pd(&hood.template coeff<0, 0, 0, 3>(x)));
1521	same1 = _mm_mul_pd(same1, _mm_load_pd(&hood.template coeff<2, 0, 0, 3>(x)));
1522	same2 = _mm_mul_pd(same2, _mm_load_pd(&hood.template coeff<4, 0, 0, 3>(x)));
1523	same3 = _mm_mul_pd(same3, _mm_load_pd(&hood.template coeff<6, 0, 0, 3>(x)));
1524
1525	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 2>(x)));
1526	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 2>(x)));
1527	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 2>(x)));
1528	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 2>(x)));
1529
1530	same0 = _mm_add_pd(same0, temp1);
1531	same1 = _mm_add_pd(same1, temp2);
1532	same2 = _mm_add_pd(same2, temp3);
1533	same3 = _mm_add_pd(same3, temp4);
1534
1535	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<0, 0, 0, 4>(x)));
1536	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<2, 0, 0, 4>(x)));
1537	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<4, 0, 0, 4>(x)));
1538	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&hood.template coeff<6, 0, 0, 4>(x)));
1539
1540	same0 = _mm_add_pd(same0, temp1);
1541	same1 = _mm_add_pd(same1, temp2);
1542	same2 = _mm_add_pd(same2, temp3);
1543	same3 = _mm_add_pd(same3, temp4);
1544
1545	neig0 = _mm_load_pd(&hood.template src<0, 0, -1>(x));
1546	neig1 = _mm_load_pd(&hood.template src<2, 0, -1>(x));
1547	neig2 = _mm_load_pd(&hood.template src<4, 0, -1>(x));
1548	neig3 = _mm_load_pd(&hood.template src<6, 0, -1>(x));
1549
1550	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 0>(x)));
1551	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 0>(x)));
1552	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 0>(x)));
1553	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 0>(x)));
1554
1555	same0 = _mm_add_pd(same0, temp1);
1556	same1 = _mm_add_pd(same1, temp2);
1557	same2 = _mm_add_pd(same2, temp3);
1558	same3 = _mm_add_pd(same3, temp4);
1559
1560	neig0 = _mm_load_pd(&hood.template src<0, -1, 0>(x));
1561	neig1 = _mm_load_pd(&hood.template src<2, -1, 0>(x));
1562	neig2 = _mm_load_pd(&hood.template src<4, -1, 0>(x));
1563	neig3 = _mm_load_pd(&hood.template src<6, -1, 0>(x));
1564
1565	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 1>(x)));
1566	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 1>(x)));
1567	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 1>(x)));
1568	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 1>(x)));
1569
1570	same0 = _mm_add_pd(same0, temp1);
1571	same1 = _mm_add_pd(same1, temp2);
1572	same2 = _mm_add_pd(same2, temp3);
1573	same3 = _mm_add_pd(same3, temp4);
1574
1575	neig0 = _mm_load_pd(&hood.template src<0, 1, 0>(x));
1576	neig1 = _mm_load_pd(&hood.template src<2, 1, 0>(x));
1577	neig2 = _mm_load_pd(&hood.template src<4, 1, 0>(x));
1578	neig3 = _mm_load_pd(&hood.template src<6, 1, 0>(x));
1579
1580	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 5>(x)));
1581	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 5>(x)));
1582	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 5>(x)));
1583	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 5>(x)));
1584
1585	same0 = _mm_add_pd(same0, temp1);
1586	same1 = _mm_add_pd(same1, temp2);
1587	same2 = _mm_add_pd(same2, temp3);
1588	same3 = _mm_add_pd(same3, temp4);
1589
1590	//xxxxxxxxxxxxx
1591	neig0 = _mm_load_pd(&hood.template src<0, 0, 1>(x));
1592	neig1 = _mm_load_pd(&hood.template src<2, 0, 1>(x));
1593	neig2 = _mm_load_pd(&hood.template src<4, 0, 1>(x));
1594	neig3 = _mm_load_pd(&hood.template src<6, 0, 1>(x));
1595
1596	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 6>(x)));
1597	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 6>(x)));
1598	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 6>(x)));
1599	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 6>(x)));
1600
1601	same0 = _mm_add_pd(same0, temp1);
1602	same1 = _mm_add_pd(same1, temp2);
1603	same2 = _mm_add_pd(same2, temp3);
1604	same3 = _mm_add_pd(same3, temp4);
1605
1606	//xxxxxxxxxxxxx
1607	neig0 = _mm_load_pd(&hood.template src<0, -1, -1>(x));
1608	neig1 = _mm_load_pd(&hood.template src<2, -1, -1>(x));
1609	neig2 = _mm_load_pd(&hood.template src<4, -1, -1>(x));
1610	neig3 = _mm_load_pd(&hood.template src<6, -1, -1>(x));
1611
1612	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 7>(x)));
1613	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 7>(x)));
1614	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 7>(x)));
1615	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 7>(x)));
1616
1617	same0 = _mm_add_pd(same0, temp1);
1618	same1 = _mm_add_pd(same1, temp2);
1619	same2 = _mm_add_pd(same2, temp3);
1620	same3 = _mm_add_pd(same3, temp4);
1621
1622	//xxxxxxxxxxxxx
1623	neig0 = _mm_load_pd(&hood.template src<0, 1, -1>(x));
1624	neig1 = _mm_load_pd(&hood.template src<2, 1, -1>(x));
1625	neig2 = _mm_load_pd(&hood.template src<4, 1, -1>(x));
1626	neig3 = _mm_load_pd(&hood.template src<6, 1, -1>(x));
1627
1628	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 8>(x)));
1629	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 8>(x)));
1630	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 8>(x)));
1631	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 8>(x)));
1632
1633	same0 = _mm_add_pd(same0, temp1);
1634	same1 = _mm_add_pd(same1, temp2);
1635	same2 = _mm_add_pd(same2, temp3);
1636	same3 = _mm_add_pd(same3, temp4);
1637
1638	//xxxxxxxxxxxxx
1639	neig0 = _mm_load_pd(&hood.template src<0, -1, 1>(x));
1640	neig1 = _mm_load_pd(&hood.template src<2, -1, 1>(x));
1641	neig2 = _mm_load_pd(&hood.template src<4, -1, 1>(x));
1642	neig3 = _mm_load_pd(&hood.template src<6, -1, 1>(x));
1643
1644	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 9>(x)));
1645	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 9>(x)));
1646	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 9>(x)));
1647	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 9>(x)));
1648
1649	same0 = _mm_add_pd(same0, temp1);
1650	same1 = _mm_add_pd(same1, temp2);
1651	same2 = _mm_add_pd(same2, temp3);
1652	same3 = _mm_add_pd(same3, temp4);
1653
1654	//xxxxxxxxxxxxx
1655	neig0 = _mm_load_pd(&hood.template src<0, 1, 1>(x));
1656	neig1 = _mm_load_pd(&hood.template src<2, 1, 1>(x));
1657	neig2 = _mm_load_pd(&hood.template src<4, 1, 1>(x));
1658	neig3 = _mm_load_pd(&hood.template src<6, 1, 1>(x));
1659
1660	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&hood.template coeff<0, 0, 0, 10>(x)));
1661	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&hood.template coeff<2, 0, 0, 10>(x)));
1662	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&hood.template coeff<4, 0, 0, 10>(x)));
1663	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&hood.template coeff<6, 0, 0, 10>(x)));
1664
1665	same0 = _mm_add_pd(same0, temp1);
1666	same1 = _mm_add_pd(same1, temp2);
1667	same2 = _mm_add_pd(same2, temp3);
1668	same3 = _mm_add_pd(same3, temp4);
1669
1670	//xxxxxxxxxxxxx
1671	neig0 = _mm_load_pd(&hood.template coeff<0, -1, -1, 11>(x));
1672	neig1 = _mm_load_pd(&hood.template coeff<2, -1, -1, 11>(x));
1673	neig2 = _mm_load_pd(&hood.template coeff<4, -1, -1, 11>(x));
1674	neig3 = _mm_load_pd(&hood.template coeff<6, -1, -1, 11>(x));
1675
1676	same0 = _mm_add_pd(same0, neig0);
1677	same1 = _mm_add_pd(same1, neig1);
1678	same2 = _mm_add_pd(same2, neig2);
1679	same3 = _mm_add_pd(same3, neig3);
1680
1681	//xxxxxxxxxxxxx
1682	neig0 = _mm_load_pd(&hood.template coeff<0, 0, -1, 11>(x));
1683	neig1 = _mm_load_pd(&hood.template coeff<2, 0, -1, 11>(x));
1684	neig2 = _mm_load_pd(&hood.template coeff<4, 0, -1, 11>(x));
1685	neig3 = _mm_load_pd(&hood.template coeff<6, 0, -1, 11>(x));
1686
1687	same0 = _mm_add_pd(same0, neig0);
1688	same1 = _mm_add_pd(same1, neig1);
1689	same2 = _mm_add_pd(same2, neig2);
1690	same3 = _mm_add_pd(same3, neig3);
1691
1692	//xxxxxxxxxxxxx
1693	neig0 = _mm_load_pd(&hood.template coeff<0, 1, -1, 11>(x));
1694	neig1 = _mm_load_pd(&hood.template coeff<2, 1, -1, 11>(x));
1695	neig2 = _mm_load_pd(&hood.template coeff<4, 1, -1, 11>(x));
1696	neig3 = _mm_load_pd(&hood.template coeff<6, 1, -1, 11>(x));
1697
1698	same0 = _mm_add_pd(same0, neig0);
1699	same1 = _mm_add_pd(same1, neig1);
1700	same2 = _mm_add_pd(same2, neig2);
1701	same3 = _mm_add_pd(same3, neig3);
1702
1703	//xxxxxxxxxxxxx
1704	neig0 = _mm_load_pd(&hood.template coeff<0, -1, 0, 11>(x));
1705	neig1 = _mm_load_pd(&hood.template coeff<2, -1, 0, 11>(x));
1706	neig2 = _mm_load_pd(&hood.template coeff<4, -1, 0, 11>(x));
1707	neig3 = _mm_load_pd(&hood.template coeff<6, -1, 0, 11>(x));
1708
1709	same0 = _mm_add_pd(same0, neig0);
1710	same1 = _mm_add_pd(same1, neig1);
1711	same2 = _mm_add_pd(same2, neig2);
1712	same3 = _mm_add_pd(same3, neig3);
1713
1714	//xxxxxxxxxxxxx
1715	neig0 = _mm_load_pd(&hood.template coeff<0, 1, 0, 11>(x));
1716	neig1 = _mm_load_pd(&hood.template coeff<2, 1, 0, 11>(x));
1717	neig2 = _mm_load_pd(&hood.template coeff<4, 1, 0, 11>(x));
1718	neig3 = _mm_load_pd(&hood.template coeff<6, 1, 0, 11>(x));
1719
1720	same0 = _mm_add_pd(same0, neig0);
1721	same1 = _mm_add_pd(same1, neig1);
1722	same2 = _mm_add_pd(same2, neig2);
1723	same3 = _mm_add_pd(same3, neig3);
1724
1725	//xxxxxxxxxxxxx
1726	neig0 = _mm_load_pd(&hood.template coeff<0, 0, 0, 11>(x));
1727	neig1 = _mm_load_pd(&hood.template coeff<2, 0, 0, 11>(x));
1728	neig2 = _mm_load_pd(&hood.template coeff<4, 0, 0, 11>(x));
1729	neig3 = _mm_load_pd(&hood.template coeff<6, 0, 0, 11>(x));
1730
1731	same0 = _mm_add_pd(same0, neig0);
1732	same1 = _mm_add_pd(same1, neig1);
1733	same2 = _mm_add_pd(same2, neig2);
1734	same3 = _mm_add_pd(same3, neig3);
1735
1736	//xxxxxxxxxxxxx
1737	neig0 = _mm_load_pd(&hood.template coeff<0, -1, 1, 11>(x));
1738	neig1 = _mm_load_pd(&hood.template coeff<2, -1, 1, 11>(x));
1739	neig2 = _mm_load_pd(&hood.template coeff<4, -1, 1, 11>(x));
1740	neig3 = _mm_load_pd(&hood.template coeff<6, -1, 1, 11>(x));
1741
1742	same0 = _mm_add_pd(same0, neig0);
1743	same1 = _mm_add_pd(same1, neig1);
1744	same2 = _mm_add_pd(same2, neig2);
1745	same3 = _mm_add_pd(same3, neig3);
1746
1747	//xxxxxxxxxxxxx
1748	neig0 = _mm_load_pd(&hood.template coeff<0, 0, 1, 11>(x));
1749	neig1 = _mm_load_pd(&hood.template coeff<2, 0, 1, 11>(x));
1750	neig2 = _mm_load_pd(&hood.template coeff<4, 0, 1, 11>(x));
1751	neig3 = _mm_load_pd(&hood.template coeff<6, 0, 1, 11>(x));
1752
1753	same0 = _mm_add_pd(same0, neig0);
1754	same1 = _mm_add_pd(same1, neig1);
1755	same2 = _mm_add_pd(same2, neig2);
1756	same3 = _mm_add_pd(same3, neig3);
1757
1758	//xxxxxxxxxxxxx
1759	neig0 = _mm_load_pd(&hood.template coeff<0, 1, 1, 11>(x));
1760	neig1 = _mm_load_pd(&hood.template coeff<2, 1, 1, 11>(x));
1761	neig2 = _mm_load_pd(&hood.template coeff<4, 1, 1, 11>(x));
1762	neig3 = _mm_load_pd(&hood.template coeff<6, 1, 1, 11>(x));
1763
1764	same0 = _mm_add_pd(same0, neig0);
1765	same1 = _mm_add_pd(same1, neig1);
1766	same2 = _mm_add_pd(same2, neig2);
1767	same3 = _mm_add_pd(same3, neig3);
1768
1769	//xxxxxxxxxxxxx
1770	neig0 = _mm_load_pd(&hood.template coeff<0, -1, -1, 12>(x));
1771	neig1 = _mm_load_pd(&hood.template coeff<2, -1, -1, 12>(x));
1772	neig2 = _mm_load_pd(&hood.template coeff<4, -1, -1, 12>(x));
1773	neig3 = _mm_load_pd(&hood.template coeff<6, -1, -1, 12>(x));
1774
1775	same0 = _mm_add_pd(same0, neig0);
1776	same1 = _mm_add_pd(same1, neig1);
1777	same2 = _mm_add_pd(same2, neig2);
1778	same3 = _mm_add_pd(same3, neig3);
1779
1780	//xxxxxxxxxxxxx
1781	neig0 = _mm_load_pd(&hood.template coeff<0, 0, -1, 12>(x));
1782	neig1 = _mm_load_pd(&hood.template coeff<2, 0, -1, 12>(x));
1783	neig2 = _mm_load_pd(&hood.template coeff<4, 0, -1, 12>(x));
1784	neig3 = _mm_load_pd(&hood.template coeff<6, 0, -1, 12>(x));
1785
1786	same0 = _mm_add_pd(same0, neig0);
1787	same1 = _mm_add_pd(same1, neig1);
1788	same2 = _mm_add_pd(same2, neig2);
1789	same3 = _mm_add_pd(same3, neig3);
1790
1791	//xxxxxxxxxxxxx
1792	neig0 = _mm_load_pd(&hood.template coeff<0, 1, -1, 12>(x));
1793	neig1 = _mm_load_pd(&hood.template coeff<2, 1, -1, 12>(x));
1794	neig2 = _mm_load_pd(&hood.template coeff<4, 1, -1, 12>(x));
1795	neig3 = _mm_load_pd(&hood.template coeff<6, 1, -1, 12>(x));
1796
1797	same0 = _mm_add_pd(same0, neig0);
1798	same1 = _mm_add_pd(same1, neig1);
1799	same2 = _mm_add_pd(same2, neig2);
1800	same3 = _mm_add_pd(same3, neig3);
1801
1802	//xxxxxxxxxxxxx
1803	neig0 = _mm_load_pd(&hood.template coeff<0, -1, 0, 12>(x));
1804	neig1 = _mm_load_pd(&hood.template coeff<2, -1, 0, 12>(x));
1805	neig2 = _mm_load_pd(&hood.template coeff<4, -1, 0, 12>(x));
1806	neig3 = _mm_load_pd(&hood.template coeff<6, -1, 0, 12>(x));
1807
1808	same0 = _mm_add_pd(same0, neig0);
1809	same1 = _mm_add_pd(same1, neig1);
1810	same2 = _mm_add_pd(same2, neig2);
1811	same3 = _mm_add_pd(same3, neig3);
1812
1813	//xxxxxxxxxxxxx
1814	neig0 = _mm_load_pd(&hood.template coeff<0, 1, 0, 12>(x));
1815	neig1 = _mm_load_pd(&hood.template coeff<2, 1, 0, 12>(x));
1816	neig2 = _mm_load_pd(&hood.template coeff<4, 1, 0, 12>(x));
1817	neig3 = _mm_load_pd(&hood.template coeff<6, 1, 0, 12>(x));
1818
1819	same0 = _mm_add_pd(same0, neig0);
1820	same1 = _mm_add_pd(same1, neig1);
1821	same2 = _mm_add_pd(same2, neig2);
1822	same3 = _mm_add_pd(same3, neig3);
1823
1824	//xxxxxxxxxxxxx
1825	neig0 = _mm_load_pd(&hood.template coeff<0, 0, 0, 12>(x));
1826	neig1 = _mm_load_pd(&hood.template coeff<2, 0, 0, 12>(x));
1827	neig2 = _mm_load_pd(&hood.template coeff<4, 0, 0, 12>(x));
1828	neig3 = _mm_load_pd(&hood.template coeff<6, 0, 0, 12>(x));
1829
1830	same0 = _mm_add_pd(same0, neig0);
1831	same1 = _mm_add_pd(same1, neig1);
1832	same2 = _mm_add_pd(same2, neig2);
1833	same3 = _mm_add_pd(same3, neig3);
1834
1835	//xxxxxxxxxxxxx
1836	neig0 = _mm_load_pd(&hood.template coeff<0, -1, 1, 12>(x));
1837	neig1 = _mm_load_pd(&hood.template coeff<2, -1, 1, 12>(x));
1838	neig2 = _mm_load_pd(&hood.template coeff<4, -1, 1, 12>(x));
1839	neig3 = _mm_load_pd(&hood.template coeff<6, -1, 1, 12>(x));
1840
1841	same0 = _mm_add_pd(same0, neig0);
1842	same1 = _mm_add_pd(same1, neig1);
1843	same2 = _mm_add_pd(same2, neig2);
1844	same3 = _mm_add_pd(same3, neig3);
1845
1846	//xxxxxxxxxxxxx
1847	neig0 = _mm_load_pd(&hood.template coeff<0, 0, 1, 12>(x));
1848	neig1 = _mm_load_pd(&hood.template coeff<2, 0, 1, 12>(x));
1849	neig2 = _mm_load_pd(&hood.template coeff<4, 0, 1, 12>(x));
1850	neig3 = _mm_load_pd(&hood.template coeff<6, 0, 1, 12>(x));
1851
1852	same0 = _mm_add_pd(same0, neig0);
1853	same1 = _mm_add_pd(same1, neig1);
1854	same2 = _mm_add_pd(same2, neig2);
1855	same3 = _mm_add_pd(same3, neig3);
1856
1857	//xxxxxxxxxxxxx
1858	neig0 = _mm_load_pd(&hood.template coeff<0, 1, 1, 12>(x));
1859	neig1 = _mm_load_pd(&hood.template coeff<2, 1, 1, 12>(x));
1860	neig2 = _mm_load_pd(&hood.template coeff<4, 1, 1, 12>(x));
1861	neig3 = _mm_load_pd(&hood.template coeff<6, 1, 1, 12>(x));
1862
1863	same0 = _mm_add_pd(same0, neig0);
1864	same1 = _mm_add_pd(same1, neig1);
1865	same2 = _mm_add_pd(same2, neig2);
1866	same3 = _mm_add_pd(same3, neig3);
1867
1868	//yyyyyyyyyyyyy
1869	_mm_store_pd(dst + x + 0, same0);
1870	_mm_store_pd(dst + x + 2, same1);
1871	_mm_store_pd(dst + x + 4, same2);
1872	_mm_store_pd(dst + x + 6, same3);
1873
1874	same0 = same4;
1875	neig0 = neig4;
1876
1877	// dst[x] =
1878	// coeff[0][x] * src[x - offsetZ] +
1879	// coeff[1][x] * src[x - offsetY] +
1880	// coeff[2][x] * src[x - 1] +
1881	// coeff[3][x] * src[x] +
1882	// coeff[4][x] * src[x + 1] +
1883	// coeff[5][x] * src[x + offsetY] +
1884	// coeff[6][x] * src[x + offsetZ];
1885	}
1886
1887	stepScalar(hood, dst, offsetY, offsetZ, x, endX);
1888	}
1889
1890	template<class NEIGHBORHOOD>
1891	inline void stepScalar(const NEIGHBORHOOD& hood, double *dst, int offsetY, int offsetZ, int startX, int endX)
1892	{
1893	for (int x = startX; x < endX; ++x) {
1894	dst[x] =
1895	hood.template coeff<0, 0, 0, 0>(x) * hood.template src<0, -1, -1>(x) +
1896	hood.template coeff<0, 0, 0, 1>(x) * hood.template src<0, 0, -1>(x) +
1897	hood.template coeff<0, 0, 0, 2>(x) * hood.template src<0, 1, -1>(x) +
1898	hood.template coeff<0, 0, 0, 3>(x) * hood.template src<0, -1, 0>(x) +
1899	hood.template coeff<0, 0, 0, 4>(x) * hood.template src<0 - 1, 0, 0>(x) +
1900	hood.template coeff<0, 0, 0, 5>(x) * hood.template src<0, 0, 0>(x) +
1901	hood.template coeff<0, 0, 0, 6>(x) * hood.template src<0 + 1, 0, 0>(x) +
1902	hood.template coeff<0, 0, 0, 7>(x) * hood.template src<0, 1, 0>(x) +
1903	hood.template coeff<0, 0, 0, 8>(x) * hood.template src<0, -1, 1>(x) +
1904	hood.template coeff<0, 0, 0, 9>(x) * hood.template src<0, 0, 1>(x) +
1905	hood.template coeff<0, 0, 0,10>(x) * hood.template src<0, 1, 1>(x) +
1906
1907	hood.template coeff<0, -1, -1, 11>(x) +
1908	hood.template coeff<0, 0, -1, 11>(x) +
1909	hood.template coeff<0, 1, -1, 11>(x) +
1910	hood.template coeff<0, -1, 0, 11>(x) +
1911	hood.template coeff<-1, 0, 0, 11>(x) +
1912	hood.template coeff<0, 0, 0, 11>(x) +
1913	hood.template coeff<1, 0, 0, 11>(x) +
1914	hood.template coeff<0, 1, 0, 11>(x) +
1915	hood.template coeff<0, -1, 1, 11>(x) +
1916	hood.template coeff<0, 0, 1, 11>(x) +
1917	hood.template coeff<0, 1, 1, 11>(x) +
1918
1919	hood.template coeff<0, -1, -1, 12>(x) +
1920	hood.template coeff<0, 0, -1, 12>(x) +
1921	hood.template coeff<0, 1, -1, 12>(x) +
1922	hood.template coeff<0, -1, 0, 12>(x) +
1923	hood.template coeff<-1, 0, 0, 12>(x) +
1924	hood.template coeff<0, 0, 0, 12>(x) +
1925	hood.template coeff<1, 0, 0, 12>(x) +
1926	hood.template coeff<0, 1, 0, 12>(x) +
1927	hood.template coeff<0, -1, 1, 12>(x) +
1928	hood.template coeff<0, 0, 1, 12>(x) +
1929	hood.template coeff<0, 1, 1, 12>(x);
1930	}
1931	}
1932
1933	int flops()
1934	{
1935	return 40;
1936	}
1937	};
1938
1939
1940	int main(int argc, char *argv[])
1941	{
1942	// Benchmark<Scalar3D, 3>().exercise();
1943	// Benchmark<Vectorized3D, 3>().exercise();
1944	// Benchmark<VectorizedSSEMelbourneShuffle2D>().exercise();
1945	// Benchmark<ExtendedVectorized3D, 3>().exercise();
1946	// Benchmark<ExtendedVectorized3DFixed, 3>().exercise();
1947	Benchmark<ExtendedVectorized3DNextGen<4, 4, MY_SIZE>, 3>().exercise();
1948	// Benchmark<Jacobi3D, 3>().exercise();
1949
1950
1951	// std::vector<cl::Platform> platforms;
1952	// cl::Platform::get(&platforms);
1953
1954	// for (int i = 0; i < platforms.size(); ++i) {
1955	// std::string str;
1956	// platforms[i].getInfo(CL_PLATFORM_NAME, &str);
1957	// std::cout << "Platform[" << i << "] = " << str << std::endl;
1958	// }
1959
1960	return 0;
1961	}

Note: See TracBrowser for help on using the browser.

Context Navigation

root/src/testbed/testbed.cpp @ 178:cc86edb738ca

Download in other formats: